hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From gunt...@apache.org
Subject svn commit: r1622597 [1/2] - in /hive/branches/tez: ./ common/src/java/org/apache/hadoop/hive/common/ common/src/java/org/apache/hadoop/hive/conf/ hcatalog/src/test/e2e/templeton/drivers/ itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/ ite...
Date Fri, 05 Sep 2014 02:32:21 GMT
Author: gunther
Date: Fri Sep  5 02:32:20 2014
New Revision: 1622597

URL: http://svn.apache.org/r1622597
Log:
Merge latest trunk into branch. (Gunther Hagleitner)

Added:
    hive/branches/tez/ql/src/test/queries/clientpositive/uber_reduce.q
      - copied unchanged from r1622596, hive/trunk/ql/src/test/queries/clientpositive/uber_reduce.q
    hive/branches/tez/ql/src/test/results/clientpositive/uber_reduce.q.out
      - copied unchanged from r1622596, hive/trunk/ql/src/test/results/clientpositive/uber_reduce.q.out
Removed:
    hive/branches/tez/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestUtilitiesDfs.java
    hive/branches/tez/service/src/test/org/apache/hive/service/cli/TestScratchDir.java
Modified:
    hive/branches/tez/   (props changed)
    hive/branches/tez/common/src/java/org/apache/hadoop/hive/common/ServerUtils.java
    hive/branches/tez/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
    hive/branches/tez/hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm
    hive/branches/tez/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java
    hive/branches/tez/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java
    hive/branches/tez/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniMr.java
    hive/branches/tez/itests/src/test/resources/testconfiguration.properties
    hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/Context.java
    hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
    hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
    hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
    hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/AcidOutputFormat.java
    hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/RecordUpdater.java
    hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcOutputFormat.java
    hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java
    hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
    hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java
    hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRawRecordMerger.java
    hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRecordUpdater.java
    hive/branches/tez/service/src/java/org/apache/hive/service/cli/CLIService.java
    hive/branches/tez/service/src/java/org/apache/hive/service/cli/session/HiveSessionBase.java
    hive/branches/tez/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
    hive/branches/tez/service/src/java/org/apache/hive/service/cli/session/SessionManager.java
    hive/branches/tez/service/src/java/org/apache/hive/service/server/HiveServer2.java

Propchange: hive/branches/tez/
------------------------------------------------------------------------------
  Merged /hive/trunk:r1622374-1622596

Modified: hive/branches/tez/common/src/java/org/apache/hadoop/hive/common/ServerUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/common/src/java/org/apache/hadoop/hive/common/ServerUtils.java?rev=1622597&r1=1622596&r2=1622597&view=diff
==============================================================================
--- hive/branches/tez/common/src/java/org/apache/hadoop/hive/common/ServerUtils.java (original)
+++ hive/branches/tez/common/src/java/org/apache/hadoop/hive/common/ServerUtils.java Fri Sep  5 02:32:20 2014
@@ -25,9 +25,7 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.conf.HiveConf;
 
 /**
- *
- * ServerUtils.
- *
+ * ServerUtils (specific to HiveServer version 1)
  */
 public class ServerUtils {
 

Modified: hive/branches/tez/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1622597&r1=1622596&r2=1622597&view=diff
==============================================================================
--- hive/branches/tez/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/branches/tez/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Fri Sep  5 02:32:20 2014
@@ -206,11 +206,18 @@ public class HiveConf extends Configurat
     PLAN_SERIALIZATION("hive.plan.serialization.format", "kryo",
         "Query plan format serialization between client and task nodes. \n" +
         "Two supported values are : kryo and javaXML. Kryo is default."),
-    SCRATCHDIR("hive.exec.scratchdir", "/tmp/hive-${system:user.name}", "Scratch space for Hive jobs"),
+    SCRATCHDIR("hive.exec.scratchdir", "/tmp/hive", 
+        "HDFS root scratch dir for Hive jobs which gets created with 777 permission. " +
+        "For each connecting user, an HDFS scratch dir: ${hive.exec.scratchdir}/<username> is created, " +
+        "with ${hive.scratch.dir.permission}."),
     LOCALSCRATCHDIR("hive.exec.local.scratchdir",
         "${system:java.io.tmpdir}" + File.separator + "${system:user.name}",
         "Local scratch space for Hive jobs"),
-    SCRATCHDIRPERMISSION("hive.scratch.dir.permission", "700", ""),
+    DOWNLOADED_RESOURCES_DIR("hive.downloaded.resources.dir",
+        "${system:java.io.tmpdir}" + File.separator + "${hive.session.id}_resources",
+        "Temporary local directory for added resources in the remote file system."),
+    SCRATCHDIRPERMISSION("hive.scratch.dir.permission", "700", 
+        "The permission for the user specific scratch directories that get created."),
     SUBMITVIACHILD("hive.exec.submitviachild", false, ""),
     SUBMITLOCALTASKVIACHILD("hive.exec.submit.local.task.via.child", true,
         "Determines whether local tasks (typically mapjoin hashtable generation phase) runs in \n" +
@@ -274,9 +281,6 @@ public class HiveConf extends Configurat
         "Maximum number of dynamic partitions allowed to be created in each mapper/reducer node."),
     MAXCREATEDFILES("hive.exec.max.created.files", 100000L,
         "Maximum number of HDFS files created by all mappers/reducers in a MapReduce job."),
-    DOWNLOADED_RESOURCES_DIR("hive.downloaded.resources.dir",
-        "${system:java.io.tmpdir}" + File.separator + "${hive.session.id}_resources",
-        "Temporary local directory for added resources in the remote file system."),
     DEFAULTPARTITIONNAME("hive.exec.default.partition.name", "__HIVE_DEFAULT_PARTITION__",
         "The default partition name in case the dynamic partition column value is null/empty string or any other values that cannot be escaped. \n" +
         "This value must not contain any special character used in HDFS URI (e.g., ':', '%', '/' etc). \n" +

Modified: hive/branches/tez/hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm?rev=1622597&r1=1622596&r2=1622597&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm (original)
+++ hive/branches/tez/hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm Fri Sep  5 02:32:20 2014
@@ -1422,7 +1422,7 @@ sub run
             $testStatuses->{$testName} = $failedStr;
 
           }
-          $msg= "$msg at " . time . "\n";
+          $msg .= "\nEnding test $testName at " . $endTime ."\n";
           #print $msg;
           print $log $msg;
           $duration = $endTime - $beginTime;
@@ -1435,6 +1435,7 @@ sub run
 
         if ($@) {
           $msg= "ERROR $subName at : ".__LINE__." Failed to run test $testName <$@>\n";
+          $msg .= "Ending test $testName at " . time ."\n";
           #print $msg;
           print $log $msg;
           $testStatuses->{$testName} = $abortedStr;

Modified: hive/branches/tez/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java?rev=1622597&r1=1622596&r2=1622597&view=diff
==============================================================================
--- hive/branches/tez/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java (original)
+++ hive/branches/tez/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java Fri Sep  5 02:32:20 2014
@@ -31,7 +31,6 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.metastore.HiveMetaStore;
 import org.apache.hadoop.hive.metastore.MetaStoreUtils;
 import org.apache.hadoop.hive.shims.HadoopShims.MiniDFSShim;
 import org.apache.hadoop.hive.shims.HadoopShims.MiniMrShim;
@@ -57,6 +56,7 @@ public class MiniHS2 extends AbstractHiv
   private static final AtomicLong hs2Counter = new AtomicLong();
   private MiniMrShim mr;
   private MiniDFSShim dfs;
+  private FileSystem localFS;
   private boolean useMiniMR = false;
   private boolean useMiniKdc = false;
   private final String serverPrincipal;
@@ -137,6 +137,10 @@ public class MiniHS2 extends AbstractHiv
     this.dfs = dfs;
   }
 
+  public FileSystem getLocalFS() {
+    return localFS;
+  }
+
   public boolean isUseMiniMR() {
     return useMiniMR;
   }
@@ -157,7 +161,8 @@ public class MiniHS2 extends AbstractHiv
     this.serverPrincipal = serverPrincipal;
     this.serverKeytab = serverKeytab;
     this.isMetastoreRemote = isMetastoreRemote;
-    baseDir =  Files.createTempDir();
+    baseDir = Files.createTempDir();
+    localFS = FileSystem.getLocal(hiveConf);
     FileSystem fs;
     if (useMiniMR) {
       dfs = ShimLoader.getHadoopShims().getMiniDfs(hiveConf, 4, true, null);
@@ -371,7 +376,7 @@ public class MiniHS2 extends AbstractHiv
           getMiniKdc().loginUser(getMiniKdc().getDefaultUserPrincipal());
           sessionConf.put("principal", serverPrincipal);
         }
-        */
+         */
         sessionHandle = hs2Client.openSession("foo", "bar", sessionConf);
       } catch (Exception e) {
         // service not started yet

Modified: hive/branches/tez/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java?rev=1622597&r1=1622596&r2=1622597&view=diff
==============================================================================
--- hive/branches/tez/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java (original)
+++ hive/branches/tez/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java Fri Sep  5 02:32:20 2014
@@ -31,7 +31,9 @@ import java.sql.Statement;
 import java.util.HashMap;
 import java.util.Map;
 
+import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hive.jdbc.miniHS2.MiniHS2;
@@ -41,255 +43,357 @@ import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
-  public class TestJdbcWithMiniHS2 {
-    private static MiniHS2 miniHS2 = null;
-    private static Path dataFilePath;
-
-    private Connection hs2Conn = null;
-
-    @BeforeClass
-    public static void beforeTest() throws Exception {
-      Class.forName(MiniHS2.getJdbcDriverName());
-      HiveConf conf = new HiveConf();
-      conf.setBoolVar(ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
-      miniHS2 = new MiniHS2(conf);
-      String dataFileDir = conf.get("test.data.files").replace('\\', '/')
-          .replace("c:", "");
-      dataFilePath = new Path(dataFileDir, "kv1.txt");
-      Map<String, String> confOverlay = new HashMap<String, String>();
-      miniHS2.start(confOverlay);
-    }
+public class TestJdbcWithMiniHS2 {
+  private static MiniHS2 miniHS2 = null;
+  private static Path dataFilePath;
+
+  private Connection hs2Conn = null;
+
+  @BeforeClass
+  public static void beforeTest() throws Exception {
+    Class.forName(MiniHS2.getJdbcDriverName());
+    HiveConf conf = new HiveConf();
+    conf.setBoolVar(ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
+    miniHS2 = new MiniHS2(conf);
+    String dataFileDir = conf.get("test.data.files").replace('\\', '/')
+        .replace("c:", "");
+    dataFilePath = new Path(dataFileDir, "kv1.txt");
+    Map<String, String> confOverlay = new HashMap<String, String>();
+    miniHS2.start(confOverlay);
+  }
 
-    @Before
-    public void setUp() throws Exception {
-      hs2Conn = getConnection(miniHS2.getJdbcURL(), System.getProperty("user.name"), "bar");
-    }
+  @Before
+  public void setUp() throws Exception {
+    hs2Conn = getConnection(miniHS2.getJdbcURL(), System.getProperty("user.name"), "bar");
+  }
 
-    private Connection getConnection(String jdbcURL, String user, String pwd) throws SQLException {
-      Connection conn = DriverManager.getConnection(jdbcURL, user, pwd);
-      conn.createStatement().execute("set hive.support.concurrency = false");
-      return conn;
-    }
+  private Connection getConnection(String jdbcURL, String user, String pwd) throws SQLException {
+    Connection conn = DriverManager.getConnection(jdbcURL, user, pwd);
+    conn.createStatement().execute("set hive.support.concurrency = false");
+    return conn;
+  }
 
-    @After
-    public void tearDown() throws Exception {
-      hs2Conn.close();
+  @After
+  public void tearDown() throws Exception {
+    hs2Conn.close();
+  }
+
+  @AfterClass
+  public static void afterTest() throws Exception {
+    if (miniHS2.isStarted()) {
+      miniHS2.stop();
     }
+  }
+
+  @Test
+  public void testConnection() throws Exception {
+    String tableName = "testTab1";
+    Statement stmt = hs2Conn.createStatement();
+
+    // create table
+    stmt.execute("DROP TABLE IF EXISTS " + tableName);
+    stmt.execute("CREATE TABLE " + tableName
+        + " (under_col INT COMMENT 'the under column', value STRING) COMMENT ' test table'");
+
+    // load data
+    stmt.execute("load data local inpath '"
+        + dataFilePath.toString() + "' into table " + tableName);
+
+    ResultSet res = stmt.executeQuery("SELECT * FROM " + tableName);
+    assertTrue(res.next());
+    assertEquals("val_238", res.getString(2));
+    res.close();
+    stmt.close();
+  }
+
 
-    @AfterClass
-    public static void afterTest() throws Exception {
-      if (miniHS2.isStarted())
-        miniHS2.stop();
+  /**   This test is to connect to any database without using the command "Use <<DB>>"
+   *  1)connect to default database.
+   *  2) Create a new DB test_default.
+   *  3) Connect to test_default database.
+   *  4) Connect and create table under test_default_test.
+   *  5) Connect and display all tables.
+   *  6) Connect to default database and shouldn't find table test_default_test.
+   *  7) Connect and drop test_default_test.
+   *  8) drop test_default database.
+   */
+
+  @Test
+  public void testURIDatabaseName() throws Exception{
+
+    String  jdbcUri  = miniHS2.getJdbcURL().substring(0, miniHS2.getJdbcURL().indexOf("default"));
+
+    hs2Conn= getConnection(jdbcUri+"default", System.getProperty("user.name"),"bar");
+    String dbName="test_connection_non_default_db";
+    String tableInNonDefaultSchema="table_in_non_default_schema";
+    Statement stmt = hs2Conn.createStatement();
+    stmt.execute("create database  if not exists "+dbName);
+    stmt.close();
+    hs2Conn.close();
+
+    hs2Conn = getConnection(jdbcUri+dbName,System.getProperty("user.name"),"bar");
+    stmt = hs2Conn .createStatement();
+    boolean expected = stmt.execute(" create table "+tableInNonDefaultSchema +" (x int)");
+    stmt.close();
+    hs2Conn .close();
+
+    hs2Conn  = getConnection(jdbcUri+dbName,System.getProperty("user.name"),"bar");
+    stmt = hs2Conn .createStatement();
+    ResultSet res = stmt.executeQuery("show tables");
+    boolean testTableExists = false;
+    while (res.next()) {
+      assertNotNull("table name is null in result set", res.getString(1));
+      if (tableInNonDefaultSchema.equalsIgnoreCase(res.getString(1))) {
+        testTableExists = true;
+      }
+    }
+    assertTrue("table name  "+tableInNonDefaultSchema
+        + "   found in SHOW TABLES result set", testTableExists);
+    stmt.close();
+    hs2Conn .close();
+
+    hs2Conn  = getConnection(jdbcUri+"default",System.getProperty("user.name"),"bar");
+    stmt = hs2Conn .createStatement();
+    res = stmt.executeQuery("show tables");
+    testTableExists = false;
+    while (res.next()) {
+      assertNotNull("table name is null in result set", res.getString(1));
+      if (tableInNonDefaultSchema.equalsIgnoreCase(res.getString(1))) {
+        testTableExists = true;
+      }
     }
 
-    @Test
-    public void testConnection() throws Exception {
-      String tableName = "testTab1";
-      Statement stmt = hs2Conn.createStatement();
-
-      // create table
-      stmt.execute("DROP TABLE IF EXISTS " + tableName);
-      stmt.execute("CREATE TABLE " + tableName
-          + " (under_col INT COMMENT 'the under column', value STRING) COMMENT ' test table'");
-
-      // load data
-      stmt.execute("load data local inpath '"
-          + dataFilePath.toString() + "' into table " + tableName);
-
-      ResultSet res = stmt.executeQuery("SELECT * FROM " + tableName);
-      assertTrue(res.next());
-      assertEquals("val_238", res.getString(2));
-      res.close();
-      stmt.close();
+    assertFalse("table name "+tableInNonDefaultSchema
+        + "  NOT  found in SHOW TABLES result set", testTableExists);
+    stmt.close();
+    hs2Conn .close();
+
+    hs2Conn  = getConnection(jdbcUri+dbName,System.getProperty("user.name"),"bar");
+    stmt = hs2Conn .createStatement();
+    stmt.execute("set hive.support.concurrency = false");
+    res = stmt.executeQuery("show tables");
+
+    stmt.execute(" drop table if exists table_in_non_default_schema");
+    expected = stmt.execute("DROP DATABASE "+ dbName);
+    stmt.close();
+
+    hs2Conn  = getConnection(jdbcUri+"default",System.getProperty("user.name"),"bar");
+    stmt = hs2Conn .createStatement();
+    res = stmt.executeQuery("show tables");
+    testTableExists = false;
+    while (res.next()) {
+      assertNotNull("table name is null in result set", res.getString(1));
+      if (tableInNonDefaultSchema.equalsIgnoreCase(res.getString(1))) {
+        testTableExists = true;
+      }
     }
 
+    // test URI with no dbName
+    hs2Conn  = getConnection(jdbcUri, System.getProperty("user.name"),"bar");
+    verifyCurrentDB("default", hs2Conn);
+    hs2Conn.close();
+
+    hs2Conn  = getConnection(jdbcUri + ";", System.getProperty("user.name"),"bar");
+    verifyCurrentDB("default", hs2Conn);
+    hs2Conn.close();
+
+    hs2Conn  = getConnection(jdbcUri + ";/foo=bar;foo1=bar1", System.getProperty("user.name"),"bar");
+    verifyCurrentDB("default", hs2Conn);
+    hs2Conn.close();
+  }
 
-    /**   This test is to connect to any database without using the command "Use <<DB>>"
-     *  1)connect to default database.
-     *  2) Create a new DB test_default.
-     *  3) Connect to test_default database.
-     *  4) Connect and create table under test_default_test.
-     *  5) Connect and display all tables.
-     *  6) Connect to default database and shouldn't find table test_default_test.
-     *  7) Connect and drop test_default_test.
-     *  8) drop test_default database.
+  @Test
+  public void testConnectionSchemaAPIs() throws Exception {
+    String db1 = "DB1";
+    /**
+     * get/set Schema are new in JDK7 and not available in java.sql.Connection in JDK6.
+     * Hence the test uses HiveConnection object to call these methods so that test will run with older JDKs
      */
+    HiveConnection hiveConn = (HiveConnection)hs2Conn;
 
-     @Test
-    public void testURIDatabaseName() throws Exception{
+    assertEquals("default", hiveConn.getSchema());
+    Statement stmt = hs2Conn.createStatement();
+    stmt.execute("DROP DATABASE IF EXISTS " + db1 + " CASCADE");
+    stmt.execute("CREATE DATABASE " + db1);
+    assertEquals("default", hiveConn.getSchema());
+
+    stmt.execute("USE " + db1);
+    assertEquals(db1, hiveConn.getSchema());
+
+    stmt.execute("USE default");
+    assertEquals("default", hiveConn.getSchema());
+
+    hiveConn.setSchema(db1);
+    assertEquals(db1, hiveConn.getSchema());
+    hiveConn.setSchema("default");
+    assertEquals("default", hiveConn.getSchema());
+
+    assertTrue(hiveConn.getCatalog().isEmpty());
+    hiveConn.setCatalog("foo");
+    assertTrue(hiveConn.getCatalog().isEmpty());
+  }
 
-     String  jdbcUri  = miniHS2.getJdbcURL().substring(0, miniHS2.getJdbcURL().indexOf("default"));
+  /**
+   * verify that the current db is the one expected. first create table as <db>.tab and then
+   * describe that table to check if <db> is the current database
+   * @param expectedDbName
+   * @param hs2Conn
+   * @throws Exception
+   */
+  private void verifyCurrentDB(String expectedDbName, Connection hs2Conn) throws Exception {
+    String verifyTab = "miniHS2DbVerificationTable";
+    Statement stmt = hs2Conn.createStatement();
+    stmt.execute("DROP TABLE IF EXISTS " + expectedDbName + "." + verifyTab);
+    stmt.execute("CREATE TABLE " + expectedDbName + "." + verifyTab + "(id INT)");
+    stmt.execute("DESCRIBE " + verifyTab);
+    stmt.execute("DROP TABLE IF EXISTS " + expectedDbName + "." + verifyTab);
+    stmt.close();
+  }
 
-     hs2Conn= getConnection(jdbcUri+"default",System.getProperty("user.name"),"bar");
-     String dbName="test_connection_non_default_db";
-     String tableInNonDefaultSchema="table_in_non_default_schema";
-     Statement stmt = hs2Conn.createStatement();
-     stmt.execute("create database  if not exists "+dbName);
-     stmt.close();
-     hs2Conn.close();
-
-     hs2Conn = getConnection(jdbcUri+dbName,System.getProperty("user.name"),"bar");
-     stmt = hs2Conn .createStatement();
-     boolean expected = stmt.execute(" create table "+tableInNonDefaultSchema +" (x int)");
-     stmt.close();
-     hs2Conn .close();
-
-     hs2Conn  = getConnection(jdbcUri+dbName,System.getProperty("user.name"),"bar");
-     stmt = hs2Conn .createStatement();
-     ResultSet res = stmt.executeQuery("show tables");
-     boolean testTableExists = false;
-     while (res.next()) {
-        assertNotNull("table name is null in result set", res.getString(1));
-        if (tableInNonDefaultSchema.equalsIgnoreCase(res.getString(1))) {
-          testTableExists = true;
-        }
-     }
-     assertTrue("table name  "+tableInNonDefaultSchema
-           + "   found in SHOW TABLES result set", testTableExists);
-     stmt.close();
-     hs2Conn .close();
-
-     hs2Conn  = getConnection(jdbcUri+"default",System.getProperty("user.name"),"bar");
-     stmt = hs2Conn .createStatement();
-     res = stmt.executeQuery("show tables");
-     testTableExists = false;
-     while (res.next()) {
-       assertNotNull("table name is null in result set", res.getString(1));
-       if (tableInNonDefaultSchema.equalsIgnoreCase(res.getString(1))) {
-         testTableExists = true;
-        }
-     }
-
-     assertFalse("table name "+tableInNonDefaultSchema
-           + "  NOT  found in SHOW TABLES result set", testTableExists);
-     stmt.close();
-     hs2Conn .close();
-
-     hs2Conn  = getConnection(jdbcUri+dbName,System.getProperty("user.name"),"bar");
-     stmt = hs2Conn .createStatement();
-     stmt.execute("set hive.support.concurrency = false");
-     res = stmt.executeQuery("show tables");
-
-     stmt.execute(" drop table if exists table_in_non_default_schema");
-     expected = stmt.execute("DROP DATABASE "+ dbName);
-     stmt.close();
-
-     hs2Conn  = getConnection(jdbcUri+"default",System.getProperty("user.name"),"bar");
-     stmt = hs2Conn .createStatement();
-     res = stmt.executeQuery("show tables");
-     testTableExists = false;
-     while (res.next()) {
-       assertNotNull("table name is null in result set", res.getString(1));
-       if (tableInNonDefaultSchema.equalsIgnoreCase(res.getString(1))) {
-         testTableExists = true;
-        }
-     }
-
-     // test URI with no dbName
-     hs2Conn  = getConnection(jdbcUri, System.getProperty("user.name"),"bar");
-     verifyCurrentDB("default", hs2Conn);
-     hs2Conn.close();
-
-     hs2Conn  = getConnection(jdbcUri + ";", System.getProperty("user.name"),"bar");
-     verifyCurrentDB("default", hs2Conn);
-     hs2Conn.close();
-
-     hs2Conn  = getConnection(jdbcUri + ";/foo=bar;foo1=bar1", System.getProperty("user.name"),"bar");
-     verifyCurrentDB("default", hs2Conn);
-     hs2Conn.close();
-     }
-
-     @Test
-     public void testConnectionSchemaAPIs() throws Exception {
-       String db1 = "DB1";
-       /**
-        * get/set Schema are new in JDK7 and not available in java.sql.Connection in JDK6.
-        * Hence the test uses HiveConnection object to call these methods so that test will run with older JDKs
-        */
-       HiveConnection hiveConn = (HiveConnection)hs2Conn;
-
-       assertEquals("default", hiveConn.getSchema());
-       Statement stmt = hs2Conn.createStatement();
-       stmt.execute("DROP DATABASE IF EXISTS " + db1 + " CASCADE");
-       stmt.execute("CREATE DATABASE " + db1);
-       assertEquals("default", hiveConn.getSchema());
-
-       stmt.execute("USE " + db1);
-       assertEquals(db1, hiveConn.getSchema());
-
-       stmt.execute("USE default");
-       assertEquals("default", hiveConn.getSchema());
-
-       hiveConn.setSchema(db1);
-       assertEquals(db1, hiveConn.getSchema());
-       hiveConn.setSchema("default");
-       assertEquals("default", hiveConn.getSchema());
-
-       assertTrue(hiveConn.getCatalog().isEmpty());
-       hiveConn.setCatalog("foo");
-       assertTrue(hiveConn.getCatalog().isEmpty());
-     }
-
-   /**
-    * verify that the current db is the one expected. first create table as <db>.tab and then
-    * describe that table to check if <db> is the current database
-    * @param expectedDbName
-    * @param hs2Conn
-    * @throws Exception
-    */
-   private void verifyCurrentDB(String expectedDbName, Connection hs2Conn) throws Exception {
-     String verifyTab = "miniHS2DbVerificationTable";
-     Statement stmt = hs2Conn.createStatement();
-     stmt.execute("DROP TABLE IF EXISTS " + expectedDbName + "." + verifyTab);
-     stmt.execute("CREATE TABLE " + expectedDbName + "." + verifyTab + "(id INT)");
-     stmt.execute("DESCRIBE " + verifyTab);
-     stmt.execute("DROP TABLE IF EXISTS " + expectedDbName + "." + verifyTab);
-     stmt.close();
-   }
-
-   /**
-    * This method tests whether while creating a new connection, the config
-    * variables specified in the JDBC URI are properly set for the connection.
-    * This is a test for HiveConnection#configureConnection.
-    *
-    * @throws Exception
-    */
-   @Test
-   public void testNewConnectionConfiguration() throws Exception {
-
-     // Set some conf parameters
-     String hiveConf = "hive.cli.print.header=true;hive.server2.async.exec.shutdown.timeout=20;"
-         + "hive.server2.async.exec.threads=30;hive.server2.thrift.http.max.worker.threads=15";
-     // Set some conf vars
-     String hiveVar = "stab=salesTable;icol=customerID";
-     String jdbcUri = miniHS2.getJdbcURL() + "?" + hiveConf + "#" + hiveVar;
-
-     // Open a new connection with these conf & vars
-     Connection con1 = DriverManager.getConnection(jdbcUri);
-
-     // Execute "set" command and retrieve values for the conf & vars specified
-     // above
-     // Assert values retrieved
-     Statement stmt = con1.createStatement();
-
-     // Verify that the property has been properly set while creating the
-     // connection above
-     verifyConfProperty(stmt, "hive.cli.print.header", "true");
-     verifyConfProperty(stmt, "hive.server2.async.exec.shutdown.timeout", "20");
-     verifyConfProperty(stmt, "hive.server2.async.exec.threads", "30");
-     verifyConfProperty(stmt, "hive.server2.thrift.http.max.worker.threads",
-         "15");
-     verifyConfProperty(stmt, "stab", "salesTable");
-     verifyConfProperty(stmt, "icol", "customerID");
-     con1.close();
-   }
-
-   private void verifyConfProperty(Statement stmt, String property,
-       String expectedValue) throws Exception {
-     ResultSet res = stmt.executeQuery("set " + property);
-     while (res.next()) {
-       String resultValues[] = res.getString(1).split("=");
-       assertEquals(resultValues[1], expectedValue);
-     }
-   }
+  /**
+   * This method tests whether while creating a new connection, the config
+   * variables specified in the JDBC URI are properly set for the connection.
+   * This is a test for HiveConnection#configureConnection.
+   *
+   * @throws Exception
+   */
+  @Test
+  public void testNewConnectionConfiguration() throws Exception {
+
+    // Set some conf parameters
+    String hiveConf = "hive.cli.print.header=true;hive.server2.async.exec.shutdown.timeout=20;"
+        + "hive.server2.async.exec.threads=30;hive.server2.thrift.http.max.worker.threads=15";
+    // Set some conf vars
+    String hiveVar = "stab=salesTable;icol=customerID";
+    String jdbcUri = miniHS2.getJdbcURL() + "?" + hiveConf + "#" + hiveVar;
+
+    // Open a new connection with these conf & vars
+    Connection con1 = DriverManager.getConnection(jdbcUri);
+
+    // Execute "set" command and retrieve values for the conf & vars specified
+    // above
+    // Assert values retrieved
+    Statement stmt = con1.createStatement();
+
+    // Verify that the property has been properly set while creating the
+    // connection above
+    verifyConfProperty(stmt, "hive.cli.print.header", "true");
+    verifyConfProperty(stmt, "hive.server2.async.exec.shutdown.timeout", "20");
+    verifyConfProperty(stmt, "hive.server2.async.exec.threads", "30");
+    verifyConfProperty(stmt, "hive.server2.thrift.http.max.worker.threads",
+        "15");
+    verifyConfProperty(stmt, "stab", "salesTable");
+    verifyConfProperty(stmt, "icol", "customerID");
+    con1.close();
+  }
+
+  private void verifyConfProperty(Statement stmt, String property,
+      String expectedValue) throws Exception {
+    ResultSet res = stmt.executeQuery("set " + property);
+    while (res.next()) {
+      String resultValues[] = res.getString(1).split("=");
+      assertEquals(resultValues[1], expectedValue);
+    }
+  }
+
+  /**
+   * Tests the creation of the 3 scratch dirs: hdfs, local, downloaded resources (which is also local).
+   * 1. Test with doAs=false: open a new JDBC session and verify the presence of directories/permissions
+   * 2. Test with doAs=true: open a new JDBC session and verify the presence of directories/permissions
+   * @throws Exception
+   */
+  @Test
+  public void testScratchDirs() throws Exception {
+    // Stop HiveServer2
+    if (miniHS2.isStarted()) {
+      miniHS2.stop();
+    }
+    HiveConf conf = new HiveConf();
+    String userName;
+    Path scratchDirPath;
+    // 1. Test with doAs=false
+    conf.setBoolean("hive.server2.enable.doAs", false);
+    // Set a custom prefix for hdfs scratch dir path
+    conf.set("hive.exec.scratchdir", "/tmp/hs2");
+    // Set a scratch dir permission
+    String fsPermissionStr = "700";
+    conf.set("hive.scratch.dir.permission", fsPermissionStr);
+    // Start an instance of HiveServer2 which uses miniMR
+    miniHS2 = new MiniHS2(conf);
+    Map<String, String> confOverlay = new HashMap<String, String>();
+    miniHS2.start(confOverlay);
+    userName = System.getProperty("user.name");
+    hs2Conn = getConnection(miniHS2.getJdbcURL(), userName, "password");
+    // FS
+    FileSystem fs = miniHS2.getLocalFS();
+
+    // Verify scratch dir paths and permission
+    // HDFS scratch dir
+    scratchDirPath = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.SCRATCHDIR) + "/" + userName);
+    verifyScratchDir(conf, fs, scratchDirPath, userName, false);
+
+    // Local scratch dir
+    scratchDirPath = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.LOCALSCRATCHDIR));
+    verifyScratchDir(conf, fs, scratchDirPath, userName, true);
+
+    // Downloaded resources dir
+    scratchDirPath = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.DOWNLOADED_RESOURCES_DIR));
+    verifyScratchDir(conf, fs, scratchDirPath, userName, true);
+
+    // 2. Test with doAs=true
+    // Restart HiveServer2 with doAs=true
+    if (miniHS2.isStarted()) {
+      miniHS2.stop();
+    }
+    conf.setBoolean("hive.server2.enable.doAs", true);
+    // Start HS2
+    miniHS2 = new MiniHS2(conf);
+    miniHS2.start(confOverlay);
+    // Test for user "neo"
+    userName = "neo";
+    hs2Conn = getConnection(miniHS2.getJdbcURL(), userName, "the-one");
+
+    // Verify scratch dir paths and permission
+    // HDFS scratch dir
+    scratchDirPath = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.SCRATCHDIR) + "/" + userName);
+    verifyScratchDir(conf, fs, scratchDirPath, userName, false);
+
+    // Local scratch dir
+    scratchDirPath = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.LOCALSCRATCHDIR));
+    verifyScratchDir(conf, fs, scratchDirPath, userName, true);
+
+    // Downloaded resources dir
+    scratchDirPath = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.DOWNLOADED_RESOURCES_DIR));
+    verifyScratchDir(conf, fs, scratchDirPath, userName, true);
+
+    // Test for user "trinity"
+    userName = "trinity";
+    hs2Conn = getConnection(miniHS2.getJdbcURL(), userName, "the-one");
+
+    // Verify scratch dir paths and permission
+    // HDFS scratch dir
+    scratchDirPath = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.SCRATCHDIR) + "/" + userName);
+    verifyScratchDir(conf, fs, scratchDirPath, userName, false);
+
+    // Local scratch dir
+    scratchDirPath = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.LOCALSCRATCHDIR));
+    verifyScratchDir(conf, fs, scratchDirPath, userName, true);
+
+    // Downloaded resources dir
+    scratchDirPath = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.DOWNLOADED_RESOURCES_DIR));
+    verifyScratchDir(conf, fs, scratchDirPath, userName, true);
+  }
+
+  private void verifyScratchDir(HiveConf conf, FileSystem fs, Path scratchDirPath,
+      String userName, boolean isLocal) throws Exception {
+    String dirType = isLocal ? "Local" : "DFS";
+    FsPermission expectedFSPermission = new FsPermission(HiveConf.getVar(conf,
+        HiveConf.ConfVars.SCRATCHDIRPERMISSION));
+    assertTrue("The expected " + dirType + " scratch dir does not exist for the user: " +
+        userName, fs.exists(scratchDirPath));
+    if (fs.exists(scratchDirPath) && !isLocal) {
+      assertEquals("DFS scratch dir permissions don't match", expectedFSPermission,
+          fs.getFileStatus(scratchDirPath).getPermission());
+    }
   }
+}
\ No newline at end of file

Modified: hive/branches/tez/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniMr.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniMr.java?rev=1622597&r1=1622596&r2=1622597&view=diff
==============================================================================
--- hive/branches/tez/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniMr.java (original)
+++ hive/branches/tez/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniMr.java Fri Sep  5 02:32:20 2014
@@ -19,16 +19,16 @@
 package org.apache.hive.jdbc;
 
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
 
-import java.util.Map;
-import java.util.HashMap;
 import java.sql.Connection;
 import java.sql.DriverManager;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.sql.Statement;
+import java.util.HashMap;
+import java.util.Map;
 
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -48,11 +48,11 @@ public class TestJdbcWithMiniMr {
   public static final String TEST_TAG = "miniHS2.miniMr.tag";
   public static final String TEST_TAG_VALUE = "miniHS2.miniMr.value";
   public static class MiniMrTestSessionHook implements HiveSessionHook {
-     @Override
-     public void run(HiveSessionHookContext sessionHookContext) throws HiveSQLException {
-       sessionHookContext.getSessionConf().set(TEST_TAG, TEST_TAG_VALUE);
-     }
-   }
+    @Override
+    public void run(HiveSessionHookContext sessionHookContext) throws HiveSQLException {
+      sessionHookContext.getSessionConf().set(TEST_TAG, TEST_TAG_VALUE);
+    }
+  }
 
   private static MiniHS2 miniHS2 = null;
   private static HiveConf conf;
@@ -93,7 +93,7 @@ public class TestJdbcWithMiniMr {
   @Before
   public void setUp() throws Exception {
     hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL(dbName),
-            System.getProperty("user.name"), "bar");
+        System.getProperty("user.name"), "bar");
     stmt = hs2Conn.createStatement();
     stmt.execute("USE " + dbName);
   }
@@ -225,7 +225,7 @@ public class TestJdbcWithMiniMr {
     String queryStr = "SELECT * FROM " + tempTableName +
         " where value = '" + resultVal + "'";
     verifyResult(queryStr, resultVal, 2);
-    
+
     // A second connection should not be able to see the table
     Connection conn2 = DriverManager.getConnection(miniHS2.getJdbcURL(dbName),
         System.getProperty("user.name"), "bar");

Modified: hive/branches/tez/itests/src/test/resources/testconfiguration.properties
URL: http://svn.apache.org/viewvc/hive/branches/tez/itests/src/test/resources/testconfiguration.properties?rev=1622597&r1=1622596&r2=1622597&view=diff
==============================================================================
--- hive/branches/tez/itests/src/test/resources/testconfiguration.properties (original)
+++ hive/branches/tez/itests/src/test/resources/testconfiguration.properties Fri Sep  5 02:32:20 2014
@@ -44,6 +44,7 @@ minimr.query.files=auto_sortmerge_join_1
   stats_counter_partitioned.q,\
   temp_table_external.q,\
   truncate_column_buckets.q,\
+  uber_reduce.q,\
   udf_using.q
 
 minitez.query.files.shared=alter_merge_2_orc.q,\

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/Context.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/Context.java?rev=1622597&r1=1622596&r2=1622597&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/Context.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/Context.java Fri Sep  5 02:32:20 2014
@@ -18,6 +18,18 @@
 
 package org.apache.hadoop.hive.ql;
 
+import java.io.DataInput;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URI;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.ConcurrentHashMap;
+
 import org.antlr.runtime.TokenRewriteStream;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -30,7 +42,6 @@ import org.apache.hadoop.fs.permission.F
 import org.apache.hadoop.hive.common.FileUtils;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.exec.TaskRunner;
-import org.apache.hadoop.hive.ql.exec.Utilities;
 import org.apache.hadoop.hive.ql.hooks.WriteEntity;
 import org.apache.hadoop.hive.ql.lockmgr.HiveLock;
 import org.apache.hadoop.hive.ql.lockmgr.HiveLockManager;
@@ -41,18 +52,6 @@ import org.apache.hadoop.hive.ql.session
 import org.apache.hadoop.hive.shims.ShimLoader;
 import org.apache.hadoop.util.StringUtils;
 
-import java.io.DataInput;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.net.URI;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.concurrent.ConcurrentHashMap;
-
 /**
  * Context for Semantic Analyzers. Usage: not reusable - construct a new one for
  * each query should call clear() at end of use to remove temporary folders
@@ -191,7 +190,7 @@ public class Context {
    * @param scratchDir path of tmp directory
    */
   private Path getScratchDir(String scheme, String authority,
-                               boolean mkdir, String scratchDir) {
+      boolean mkdir, String scratchDir) {
 
     String fileSystem =  scheme + ":" + authority;
     Path dir = fsScratchDirs.get(fileSystem + "-" + TaskRunner.getTaskRunnerID());
@@ -203,11 +202,11 @@ public class Context {
         try {
           FileSystem fs = dirPath.getFileSystem(conf);
           dirPath = new Path(fs.makeQualified(dirPath).toString());
-          FsPermission fsPermission = new FsPermission(Short.parseShort(scratchDirPermission.trim(), 8));
+          FsPermission fsPermission = new FsPermission(scratchDirPermission);
 
-          if (!Utilities.createDirsWithPermission(conf, dirPath, fsPermission)) {
+          if (!fs.mkdirs(dirPath, fsPermission)) {
             throw new RuntimeException("Cannot make directory: "
-                                       + dirPath.toString());
+                + dirPath.toString());
           }
           if (isHDFSCleanup) {
             fs.deleteOnExit(dirPath);
@@ -233,7 +232,7 @@ public class Context {
       FileSystem fs = FileSystem.getLocal(conf);
       URI uri = fs.getUri();
       return getScratchDir(uri.getScheme(), uri.getAuthority(),
-                           mkdir, localScratchDir);
+          mkdir, localScratchDir);
     } catch (IOException e) {
       throw new RuntimeException (e);
     }
@@ -257,7 +256,7 @@ public class Context {
       URI uri = dir.toUri();
 
       Path newScratchDir = getScratchDir(uri.getScheme(), uri.getAuthority(),
-                           !explain, uri.getPath());
+          !explain, uri.getPath());
       LOG.info("New scratch dir is " + newScratchDir);
       return newScratchDir;
     } catch (IOException e) {
@@ -270,7 +269,7 @@ public class Context {
 
   private Path getExternalScratchDir(URI extURI) {
     return getScratchDir(extURI.getScheme(), extURI.getAuthority(),
-                         !explain, nonLocalScratchPath.toUri().getPath());
+        !explain, nonLocalScratchPath.toUri().getPath());
   }
 
   /**
@@ -283,7 +282,7 @@ public class Context {
         p.getFileSystem(conf).delete(p, true);
       } catch (Exception e) {
         LOG.warn("Error Removing Scratch: "
-                 + StringUtils.stringifyException(e));
+            + StringUtils.stringifyException(e));
       }
     }
     fsScratchDirs.clear();
@@ -305,7 +304,7 @@ public class Context {
    */
   public boolean isMRTmpFileURI(String uriStr) {
     return (uriStr.indexOf(executionId) != -1) &&
-      (uriStr.indexOf(MR_PREFIX) != -1);
+        (uriStr.indexOf(MR_PREFIX) != -1);
   }
 
   /**
@@ -315,7 +314,7 @@ public class Context {
    */
   public Path getMRTmpPath() {
     return new Path(getMRScratchDir(), MR_PREFIX +
-      nextPathId());
+        nextPathId());
   }
 
   /**
@@ -343,7 +342,7 @@ public class Context {
       return getExtTmpPathRelTo(path.getParent());
     }
     return new Path(getExternalScratchDir(extURI), EXT_PREFIX +
-      nextPathId());
+        nextPathId());
   }
 
   /**
@@ -353,8 +352,8 @@ public class Context {
    */
   public Path getExtTmpPathRelTo(Path path) {
     URI uri = path.toUri();
-    return new Path (getScratchDir(uri.getScheme(), uri.getAuthority(), !explain, 
-    uri.getPath() + Path.SEPARATOR + "_" + this.executionId), EXT_PREFIX + nextPathId());
+    return new Path (getScratchDir(uri.getScheme(), uri.getAuthority(), !explain,
+        uri.getPath() + Path.SEPARATOR + "_" + this.executionId), EXT_PREFIX + nextPathId());
   }
 
   /**

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java?rev=1622597&r1=1622596&r2=1622597&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java Fri Sep  5 02:32:20 2014
@@ -4017,6 +4017,9 @@ public class DDLTask extends Task<DDLWor
       tbl.setDbName(names[0]);
       tbl.setTableName(names[1]);
 
+      // using old table object, hence reset the owner to current user for new table.
+      tbl.setOwner(SessionState.getUserFromAuthenticator());
+
       if (crtTbl.getLocation() != null) {
         tbl.setDataLocation(new Path(crtTbl.getLocation()));
       } else {

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java?rev=1622597&r1=1622596&r2=1622597&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java Fri Sep  5 02:32:20 2014
@@ -92,7 +92,6 @@ import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.PathFilter;
-import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hive.common.HiveInterruptCallback;
 import org.apache.hadoop.hive.common.HiveInterruptUtils;
 import org.apache.hadoop.hive.common.HiveStatsUtils;
@@ -326,7 +325,9 @@ public final class Utilities {
       assert path != null;
       if (!gWorkMap.containsKey(path)) {
         Path localPath;
-        if (ShimLoader.getHadoopShims().isLocalMode(conf)) {
+        if (conf.getBoolean("mapreduce.task.uberized", false) && name.equals(REDUCE_PLAN_NAME)) {
+          localPath = new Path(name);
+        } else if (ShimLoader.getHadoopShims().isLocalMode(conf)) {
           localPath = path;
         } else {
           localPath = new Path(name);
@@ -3355,7 +3356,6 @@ public final class Utilities {
   private static void createTmpDirs(Configuration conf,
       List<Operator<? extends OperatorDesc>> ops) throws IOException {
 
-    FsPermission fsPermission = new FsPermission((short)00777);
     while (!ops.isEmpty()) {
       Operator<? extends OperatorDesc> op = ops.remove(0);
 
@@ -3365,7 +3365,8 @@ public final class Utilities {
 
         if (tempDir != null) {
           Path tempPath = Utilities.toTempPath(tempDir);
-          createDirsWithPermission(conf, tempPath, fsPermission);
+          FileSystem fs = tempPath.getFileSystem(conf);
+          fs.mkdirs(tempPath);
         }
       }
 
@@ -3501,76 +3502,6 @@ public final class Utilities {
   }
 
   /**
-   * @param conf the configuration used to derive the filesystem to create the path
-   * @param mkdir the path to be created
-   * @param fsPermission ignored if it is hive server session and doAs is enabled
-   * @return true if successfully created the directory else false
-   * @throws IOException if hdfs experiences any error conditions
-   */
-  public static boolean createDirsWithPermission(Configuration conf, Path mkdir,
-      FsPermission fsPermission) throws IOException {
-
-    boolean recursive = false;
-    if (SessionState.get() != null) {
-      recursive = SessionState.get().isHiveServerQuery() &&
-          conf.getBoolean(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS.varname,
-              HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS.defaultBoolVal);
-      // we reset the permission in case of hive server and doAs enabled because
-      // currently scratch directory uses /tmp/hive-hive as the scratch directory.
-      // However, with doAs enabled, the first user to create this directory would
-      // own the directory and subsequent users cannot access the scratch directory.
-      // The right fix is to have scratch dir per user.
-      fsPermission = new FsPermission((short)00777);
-    }
-
-    // if we made it so far without exception we are good!
-    return createDirsWithPermission(conf, mkdir, fsPermission, recursive);
-  }
-
-  private static void resetConfAndCloseFS (Configuration conf, boolean unsetUmask,
-      String origUmask, FileSystem fs) throws IOException {
-    if (unsetUmask) {
-      if (origUmask != null) {
-        conf.set(FsPermission.UMASK_LABEL, origUmask);
-      } else {
-        conf.unset(FsPermission.UMASK_LABEL);
-      }
-    }
-
-    fs.close();
-  }
-
-  public static boolean createDirsWithPermission(Configuration conf, Path mkdirPath,
-      FsPermission fsPermission, boolean recursive) throws IOException {
-    String origUmask = null;
-    LOG.debug("Create dirs " + mkdirPath + " with permission " + fsPermission + " recursive " +
-        recursive);
-
-    if (recursive) {
-      origUmask = conf.get(FsPermission.UMASK_LABEL);
-      // this umask is required because by default the hdfs mask is 022 resulting in
-      // all parents getting the fsPermission & !(022) permission instead of fsPermission
-      conf.set(FsPermission.UMASK_LABEL, "000");
-    }
-
-    FileSystem fs = ShimLoader.getHadoopShims().getNonCachedFileSystem(mkdirPath.toUri(), conf);
-    boolean retval = false;
-    try {
-      retval = fs.mkdirs(mkdirPath, fsPermission);
-      resetConfAndCloseFS(conf, recursive, origUmask, fs);
-    } catch (IOException ioe) {
-      try {
-        resetConfAndCloseFS(conf, recursive, origUmask, fs);
-      }
-      catch (IOException e) {
-        // do nothing - double failure
-      }
-    }
-    return retval;
-  }
-
-
-  /**
    * Convert path to qualified path.
    *
    * @param conf

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java?rev=1622597&r1=1622596&r2=1622597&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java Fri Sep  5 02:32:20 2014
@@ -43,7 +43,7 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.session.SessionState;
 import org.apache.hadoop.hive.shims.ShimLoader;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.yarn.api.records.LocalResource;
@@ -308,11 +308,11 @@ public class TezSessionState {
     throws IOException {
 
     // tez needs its own scratch dir (per session)
-    Path tezDir = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.SCRATCHDIR), TEZ_DIR);
+    Path tezDir = new Path(SessionState.get().getHdfsScratchDirURIString(), TEZ_DIR);
     tezDir = new Path(tezDir, sessionId);
     FileSystem fs = tezDir.getFileSystem(conf);
-    FsPermission fsPermission = new FsPermission((short)00777);
-    Utilities.createDirsWithPermission(conf, tezDir, fsPermission, true);
+    FsPermission fsPermission = new FsPermission(HiveConf.getVar(conf, HiveConf.ConfVars.SCRATCHDIRPERMISSION));
+    fs.mkdirs(tezDir, fsPermission);
     // Make sure the path is normalized (we expect validation to pass since we just created it).
     tezDir = DagUtils.validateTargetDir(tezDir, conf).getPath();
     // don't keep the directory around on non-clean exit

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/AcidOutputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/AcidOutputFormat.java?rev=1622597&r1=1622596&r2=1622597&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/AcidOutputFormat.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/AcidOutputFormat.java Fri Sep  5 02:32:20 2014
@@ -52,6 +52,7 @@ public interface AcidOutputFormat<K exte
     private int bucket;
     private PrintStream dummyStream = null;
     private boolean oldStyle = false;
+    private int recIdCol = -1;  // Column the record identifier is in, -1 indicates no record id
 
     /**
      * Create the options object.
@@ -164,6 +165,16 @@ public interface AcidOutputFormat<K exte
     }
 
     /**
+     * Which column the row id field is in.
+     * @param recIdCol
+     * @return this
+     */
+    public Options recordIdColumn(int recIdCol) {
+      this.recIdCol = recIdCol;
+      return this;
+    }
+
+    /**
      * Temporary switch while we are in development that replaces the
      * implementation with a dummy one that just prints to stream.
      * @param stream the stream to print to
@@ -214,6 +225,10 @@ public interface AcidOutputFormat<K exte
       return bucket;
     }
 
+    public int getRecordIdColumn() {
+      return recIdCol;
+    }
+
     public PrintStream getDummyStream() {
       return dummyStream;
     }

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/RecordUpdater.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/RecordUpdater.java?rev=1622597&r1=1622596&r2=1622597&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/RecordUpdater.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/RecordUpdater.java Fri Sep  5 02:32:20 2014
@@ -40,26 +40,17 @@ public interface RecordUpdater {
   /**
    * Update an old record with a new set of values.
    * @param currentTransaction the current transaction id
-   * @param originalTransaction the row's original transaction id
-   * @param rowId the original row id
    * @param row the new values for the row
    * @throws IOException
    */
-  void update(long currentTransaction,
-              long originalTransaction,
-              long rowId,
-              Object row) throws IOException;
+  void update(long currentTransaction, Object row) throws IOException;
 
   /**
    * Delete a row from the table.
    * @param currentTransaction the current transaction id
-   * @param originalTransaction the rows original transaction id
-   * @param rowId the original row id
    * @throws IOException
    */
-  void delete(long currentTransaction,
-              long originalTransaction,
-              long rowId) throws IOException;
+  void delete(long currentTransaction, Object row) throws IOException;
 
   /**
    * Flush the current set of rows to the underlying file system, so that

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcOutputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcOutputFormat.java?rev=1622597&r1=1622596&r2=1622597&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcOutputFormat.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcOutputFormat.java Fri Sep  5 02:32:20 2014
@@ -211,18 +211,14 @@ public class OrcOutputFormat extends Fil
     }
 
     @Override
-    public void update(long currentTransaction, long originalTransaction,
-                       long rowId, Object row) throws IOException {
+    public void update(long currentTransaction, Object row) throws IOException {
       out.println("update " + path + " currTxn: " + currentTransaction +
-          " origTxn: " + originalTransaction + " row: " + rowId + " obj: " +
-          stringifyObject(row, inspector));
+          " obj: " + stringifyObject(row, inspector));
     }
 
     @Override
-    public void delete(long currentTransaction, long originalTransaction,
-                       long rowId) throws IOException {
-      out.println("delete " + path + " currTxn: " + currentTransaction +
-         " origTxn: " + originalTransaction + " row: " + rowId);
+    public void delete(long currentTransaction, Object row) throws IOException {
+      out.println("delete " + path + " currTxn: " + currentTransaction + " obj: " + row);
     }
 
     @Override

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java?rev=1622597&r1=1622596&r2=1622597&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java Fri Sep  5 02:32:20 2014
@@ -31,18 +31,18 @@ import org.apache.hadoop.hive.ql.io.Reco
 import org.apache.hadoop.hive.serde2.SerDeStats;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.LongWritable;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
-import java.nio.CharBuffer;
 import java.nio.charset.CharacterCodingException;
 import java.nio.charset.Charset;
 import java.nio.charset.CharsetDecoder;
 
-import java.nio.charset.CharsetEncoder;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -92,6 +92,14 @@ public class OrcRecordUpdater implements
   // because that is monotonically increasing to give new unique row ids.
   private long rowCountDelta = 0;
   private final KeyIndexBuilder indexBuilder = new KeyIndexBuilder();
+  private StructField recIdField = null; // field to look for the record identifier in
+  private StructField rowIdField = null; // field inside recId to look for row id in
+  private StructField originalTxnField = null;  // field inside recId to look for original txn in
+  private StructObjectInspector rowInspector; // OI for the original row
+  private StructObjectInspector recIdInspector; // OI for the record identifier struct
+  private LongObjectInspector rowIdInspector; // OI for the long row id inside the recordIdentifier
+  private LongObjectInspector origTxnInspector; // OI for the original txn inside the record
+  // identifer
 
   static class AcidStats {
     long inserts;
@@ -179,7 +187,7 @@ public class OrcRecordUpdater implements
    * @param rowInspector the row's object inspector
    * @return an object inspector for the event stream
    */
-  static ObjectInspector createEventSchema(ObjectInspector rowInspector) {
+  static StructObjectInspector createEventSchema(ObjectInspector rowInspector) {
     List<StructField> fields = new ArrayList<StructField>();
     fields.add(new OrcStruct.Field("operation",
         PrimitiveObjectInspectorFactory.writableIntObjectInspector, OPERATION));
@@ -237,7 +245,9 @@ public class OrcRecordUpdater implements
       writerOptions.bufferSize(DELTA_BUFFER_SIZE);
       writerOptions.stripeSize(DELTA_STRIPE_SIZE);
     }
-    writerOptions.inspector(createEventSchema(options.getInspector()));
+    rowInspector = (StructObjectInspector)options.getInspector();
+    writerOptions.inspector(createEventSchema(findRecId(options.getInspector(),
+        options.getRecordIdColumn())));
     this.writer = OrcFile.createWriter(this.path, writerOptions);
     item = new OrcStruct(FIELDS);
     item.setFieldValue(OPERATION, operation);
@@ -247,14 +257,50 @@ public class OrcRecordUpdater implements
     item.setFieldValue(ROW_ID, rowId);
   }
 
-  private void addEvent(int operation, long currentTransaction,
-                        long originalTransaction, long rowId,
-                        Object row) throws IOException {
+  // Find the record identifier column (if there) and return a possibly new ObjectInspector that
+  // will strain out the record id for the underlying writer.
+  private ObjectInspector findRecId(ObjectInspector inspector, int rowIdColNum) {
+    if (!(inspector instanceof StructObjectInspector)) {
+      throw new RuntimeException("Serious problem, expected a StructObjectInspector, but got a " +
+          inspector.getClass().getName());
+    }
+    if (rowIdColNum < 0) {
+      return inspector;
+    } else {
+      RecIdStrippingObjectInspector newInspector =
+          new RecIdStrippingObjectInspector(inspector, rowIdColNum);
+      recIdField = newInspector.getRecId();
+      List<? extends StructField> fields =
+          ((StructObjectInspector) recIdField.getFieldObjectInspector()).getAllStructFieldRefs();
+      // Go by position, not field name, as field names aren't guaranteed.  The order of fields
+      // in RecordIdentifier is transactionId, bucketId, rowId
+      originalTxnField = fields.get(0);
+      origTxnInspector = (LongObjectInspector)originalTxnField.getFieldObjectInspector();
+      rowIdField = fields.get(2);
+      rowIdInspector = (LongObjectInspector)rowIdField.getFieldObjectInspector();
+
+
+      recIdInspector = (StructObjectInspector) recIdField.getFieldObjectInspector();
+      return newInspector;
+    }
+  }
+
+  private void addEvent(int operation, long currentTransaction, long rowId, Object row)
+      throws IOException {
     this.operation.set(operation);
     this.currentTransaction.set(currentTransaction);
-    this.originalTransaction.set(originalTransaction);
+    // If this is an insert, originalTransaction should be set to this transaction.  If not,
+    // it will be reset by the following if anyway.
+    long originalTransaction = currentTransaction;
+    if (operation == DELETE_OPERATION || operation == UPDATE_OPERATION) {
+      Object rowIdValue = rowInspector.getStructFieldData(row, recIdField);
+      originalTransaction = origTxnInspector.get(
+          recIdInspector.getStructFieldData(rowIdValue, originalTxnField));
+      rowId = rowIdInspector.get(recIdInspector.getStructFieldData(rowIdValue, rowIdField));
+    }
     this.rowId.set(rowId);
-    item.setFieldValue(OrcRecordUpdater.ROW, row);
+    this.originalTransaction.set(originalTransaction);
+    item.setFieldValue(OrcRecordUpdater.ROW, (operation == DELETE_OPERATION ? null : row));
     indexBuilder.addKey(operation, originalTransaction, bucket.get(), rowId);
     writer.addRow(item);
   }
@@ -264,30 +310,26 @@ public class OrcRecordUpdater implements
     if (this.currentTransaction.get() != currentTransaction) {
       insertedRows = 0;
     }
-    addEvent(INSERT_OPERATION, currentTransaction, currentTransaction,
-        insertedRows++, row);
+    addEvent(INSERT_OPERATION, currentTransaction, insertedRows++, row);
     rowCountDelta++;
   }
 
   @Override
-  public void update(long currentTransaction, long originalTransaction,
-                     long rowId, Object row) throws IOException {
+  public void update(long currentTransaction, Object row) throws IOException {
     if (this.currentTransaction.get() != currentTransaction) {
       insertedRows = 0;
     }
-    addEvent(UPDATE_OPERATION, currentTransaction, originalTransaction, rowId,
-        row);
+    addEvent(UPDATE_OPERATION, currentTransaction, -1L, row);
   }
 
   @Override
-  public void delete(long currentTransaction, long originalTransaction,
-                     long rowId) throws IOException {
+  public void delete(long currentTransaction, Object row) throws IOException {
     if (this.currentTransaction.get() != currentTransaction) {
       insertedRows = 0;
     }
-    addEvent(DELETE_OPERATION, currentTransaction, originalTransaction, rowId,
-        null);
+    addEvent(DELETE_OPERATION, currentTransaction, -1, row);
     rowCountDelta--;
+
   }
 
   @Override
@@ -311,7 +353,7 @@ public class OrcRecordUpdater implements
         fs.delete(path, false);
       }
     } else {
-      writer.close();
+      if (writer != null) writer.close();
     }
     if (flushLengths != null) {
       flushLengths.close();
@@ -406,4 +448,67 @@ public class OrcRecordUpdater implements
       lastRowId = rowId;
     }
   }
+
+  /**
+   * An ObjectInspector that will strip out the record identifier so that the underlying writer
+   * doesn't see it.
+   */
+  private static class RecIdStrippingObjectInspector extends StructObjectInspector {
+    private StructObjectInspector wrapped;
+    List<StructField> fields;
+    StructField recId;
+
+    RecIdStrippingObjectInspector(ObjectInspector oi, int rowIdColNum) {
+      if (!(oi instanceof StructObjectInspector)) {
+        throw new RuntimeException("Serious problem, expected a StructObjectInspector, " +
+            "but got a " + oi.getClass().getName());
+      }
+      wrapped = (StructObjectInspector)oi;
+      List<? extends StructField> wrappedFields = wrapped.getAllStructFieldRefs();
+      fields = new ArrayList<StructField>(wrapped.getAllStructFieldRefs().size());
+      for (int i = 0; i < wrappedFields.size(); i++) {
+        if (i == rowIdColNum) {
+          recId = wrappedFields.get(i);
+        } else {
+          fields.add(wrappedFields.get(i));
+        }
+      }
+    }
+
+    @Override
+    public List<? extends StructField> getAllStructFieldRefs() {
+      return fields;
+    }
+
+    @Override
+    public StructField getStructFieldRef(String fieldName) {
+      return wrapped.getStructFieldRef(fieldName);
+    }
+
+    @Override
+    public Object getStructFieldData(Object data, StructField fieldRef) {
+      // For performance don't check that that the fieldRef isn't recId everytime,
+      // just assume that the caller used getAllStructFieldRefs and thus doesn't have that fieldRef
+      return wrapped.getStructFieldData(data, fieldRef);
+    }
+
+    @Override
+    public List<Object> getStructFieldsDataAsList(Object data) {
+      return wrapped.getStructFieldsDataAsList(data);
+    }
+
+    @Override
+    public String getTypeName() {
+      return wrapped.getTypeName();
+    }
+
+    @Override
+    public Category getCategory() {
+      return wrapped.getCategory();
+    }
+
+    StructField getRecId() {
+      return recId;
+    }
+  }
 }

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java?rev=1622597&r1=1622596&r2=1622597&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java Fri Sep  5 02:32:20 2014
@@ -45,7 +45,6 @@ import org.apache.hadoop.fs.permission.F
 import org.apache.hadoop.hive.common.JavaUtils;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
 import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
 import org.apache.hadoop.hive.ql.MapRedStats;
 import org.apache.hadoop.hive.ql.exec.Utilities;
@@ -69,6 +68,7 @@ import org.apache.hadoop.hive.ql.securit
 import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveMetastoreClientFactoryImpl;
 import org.apache.hadoop.hive.ql.util.DosToUnix;
 import org.apache.hadoop.hive.shims.ShimLoader;
+import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.ReflectionUtils;
 
 import com.google.common.base.Preconditions;
@@ -208,6 +208,8 @@ public class SessionState {
    */
   private Path localSessionPath;
 
+  private String hdfsScratchDirURIString;
+
   /**
    * Get the lineage state stored in this session.
    *
@@ -344,36 +346,39 @@ public class SessionState {
 
     setCurrentSessionState(startSs);
 
-    if(startSs.hiveHist == null){
+    if (startSs.hiveHist == null){
       if (startSs.getConf().getBoolVar(HiveConf.ConfVars.HIVE_SESSION_HISTORY_ENABLED)) {
         startSs.hiveHist = new HiveHistoryImpl(startSs);
-      }else {
-        //Hive history is disabled, create a no-op proxy
+      } else {
+        // Hive history is disabled, create a no-op proxy
         startSs.hiveHist = HiveHistoryProxyHandler.getNoOpHiveHistoryProxy();
       }
     }
 
-    if (startSs.getTmpOutputFile() == null) {
-      // set temp file containing results to be sent to HiveClient
-      try {
-        startSs.setTmpOutputFile(createTempFile(startSs.getConf()));
-      } catch (IOException e) {
-        throw new RuntimeException(e);
-      }
-    }
-
     // Get the following out of the way when you start the session these take a
     // while and should be done when we start up.
     try {
-      //Hive object instance should be created with a copy of the conf object. If the conf is
+      // Hive object instance should be created with a copy of the conf object. If the conf is
       // shared with SessionState, other parts of the code might update the config, but
       // Hive.get(HiveConf) would not recognize the case when it needs refreshing
       Hive.get(new HiveConf(startSs.conf)).getMSC();
-      ShimLoader.getHadoopShims().getUGIForConf(startSs.conf);
+      UserGroupInformation sessionUGI = ShimLoader.getHadoopShims().getUGIForConf(startSs.conf);
       FileSystem.get(startSs.conf);
-      startSs.createSessionPaths(startSs.conf);
+
+      // Create scratch dirs for this session
+      startSs.createSessionDirs(sessionUGI.getShortUserName());
+
+      // Set temp file containing results to be sent to HiveClient
+      if (startSs.getTmpOutputFile() == null) {
+        try {
+          startSs.setTmpOutputFile(createTempFile(startSs.getConf()));
+        } catch (IOException e) {
+          throw new RuntimeException(e);
+        }
+      }
+
     } catch (Exception e) {
-      // catch-all due to some exec time dependencies on session state
+      // Catch-all due to some exec time dependencies on session state
       // that would cause ClassNoFoundException otherwise
       throw new RuntimeException(e);
     }
@@ -396,6 +401,88 @@ public class SessionState {
     return startSs;
   }
 
+  /**
+   * Create dirs & session paths for this session:
+   * 1. HDFS scratch dir
+   * 2. Local scratch dir
+   * 3. Local downloaded resource dir
+   * 4. HDFS session path
+   * 5. Local session path
+   * 6. HDFS temp table space
+   * @param userName
+   * @throws IOException
+   */
+  private void createSessionDirs(String userName) throws IOException {
+    HiveConf conf = getConf();
+    // First create the root scratch dir on hdfs (if it doesn't already exist) and make it writable
+    Path rootHDFSDirPath = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.SCRATCHDIR));
+    String rootHDFSDirPermission = "777";
+    createPath(conf, rootHDFSDirPath, rootHDFSDirPermission, false, false);
+    // Now create session specific dirs
+    String scratchDirPermission = HiveConf.getVar(conf, HiveConf.ConfVars.SCRATCHDIRPERMISSION);
+    Path path;
+    // 1. HDFS scratch dir
+    path = new Path(rootHDFSDirPath, userName);
+    hdfsScratchDirURIString = path.toUri().toString();
+    createPath(conf, path, scratchDirPermission, false, false);
+    // 2. Local scratch dir
+    path = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.LOCALSCRATCHDIR));
+    createPath(conf, path, scratchDirPermission, true, false);
+    // 3. Download resources dir
+    path = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.DOWNLOADED_RESOURCES_DIR));
+    createPath(conf, path, scratchDirPermission, true, false);
+    // Finally, create session paths for this session
+    // Local & non-local tmp location is configurable. however it is the same across
+    // all external file systems
+    String sessionId = getSessionId();
+    // 4. HDFS session path
+    hdfsSessionPath = new Path(hdfsScratchDirURIString, sessionId);
+    createPath(conf, hdfsSessionPath, scratchDirPermission, false, true);
+    conf.set(HDFS_SESSION_PATH_KEY, hdfsSessionPath.toUri().toString());
+    // 5. Local session path
+    localSessionPath = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.LOCALSCRATCHDIR), sessionId);
+    createPath(conf, localSessionPath, scratchDirPermission, true, true);
+    conf.set(LOCAL_SESSION_PATH_KEY, localSessionPath.toUri().toString());
+    // 6. HDFS temp table space
+    hdfsTmpTableSpace = new Path(hdfsSessionPath, TMP_PREFIX);
+    createPath(conf, hdfsTmpTableSpace, scratchDirPermission, false, true);
+    conf.set(TMP_TABLE_SPACE_KEY, hdfsTmpTableSpace.toUri().toString());
+  }
+
+  /**
+   * Create a given path if it doesn't exist.
+   *
+   * @param conf
+   * @param pathString
+   * @param permission
+   * @param isLocal
+   * @param isCleanUp
+   * @return
+   * @throws IOException
+   */
+  private void createPath(HiveConf conf, Path path, String permission, boolean isLocal,
+      boolean isCleanUp) throws IOException {
+    FsPermission fsPermission = new FsPermission(permission);
+    FileSystem fs;
+    if (isLocal) {
+      fs = FileSystem.getLocal(conf);
+    } else {
+      fs = path.getFileSystem(conf);
+    }
+    if (!fs.exists(path)) {
+      fs.mkdirs(path, fsPermission);
+      String dirType = isLocal ? "local" : "HDFS";
+      LOG.info("Created " + dirType + " directory: " + path.toString());
+    }
+    if (isCleanUp) {
+      fs.deleteOnExit(path);
+    }
+  }
+
+  public String getHdfsScratchDirURIString() {
+    return hdfsScratchDirURIString;
+  }
+
   public static Path getLocalSessionPath(Configuration conf) {
     SessionState ss = SessionState.get();
     if (ss == null) {
@@ -448,43 +535,6 @@ public class SessionState {
     }
   }
 
-  private void createSessionPaths(Configuration conf) throws IOException {
-
-    String scratchDirPermission = HiveConf.getVar(conf, HiveConf.ConfVars.SCRATCHDIRPERMISSION);
-    String sessionId = getSessionId();
-
-    // local & non-local tmp location is configurable. however it is the same across
-    // all external file systems
-    hdfsSessionPath =
-      new Path(HiveConf.getVar(conf, HiveConf.ConfVars.SCRATCHDIR),
-               sessionId);
-    createPath(conf, hdfsSessionPath, scratchDirPermission);
-    conf.set(HDFS_SESSION_PATH_KEY, hdfsSessionPath.toUri().toString());
-
-    localSessionPath = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.LOCALSCRATCHDIR),
-                                sessionId);
-    createPath(conf, localSessionPath, scratchDirPermission);
-    conf.set(LOCAL_SESSION_PATH_KEY, localSessionPath.toUri().toString());
-    hdfsTmpTableSpace = new Path(hdfsSessionPath, TMP_PREFIX);
-    createPath(conf, hdfsTmpTableSpace, scratchDirPermission);
-    conf.set(TMP_TABLE_SPACE_KEY, hdfsTmpTableSpace.toUri().toString());
-  }
-
-  private void createPath(Configuration conf, Path p, String perm) throws IOException {
-    FileSystem fs = p.getFileSystem(conf);
-    p = new Path(fs.makeQualified(p).toString());
-    FsPermission fsPermission = new FsPermission(Short.parseShort(perm.trim(), 8));
-
-    if (!Utilities.createDirsWithPermission(conf, p, fsPermission)) {
-      throw new IOException("Cannot create directory: "
-                            + p.toString());
-    }
-
-    // best effort to clean up if we don't shut down properly
-    fs.deleteOnExit(p);
-  }
-
-
   /**
    * Setup authentication and authorization plugins for this session.
    */

Modified: hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java?rev=1622597&r1=1622596&r2=1622597&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java (original)
+++ hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java Fri Sep  5 02:32:20 2014
@@ -20,15 +20,12 @@ package org.apache.hadoop.hive.ql.exec;
 
 import static org.apache.hadoop.hive.ql.exec.Utilities.getFileExtension;
 
-import java.io.IOException;
 import java.sql.Timestamp;
 import java.util.ArrayList;
 import java.util.List;
 
 import junit.framework.TestCase;
 
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -39,7 +36,6 @@ import org.apache.hadoop.hive.ql.session
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDFFromUtcTimestamp;
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
 import org.apache.hadoop.mapred.JobConf;
-import org.junit.Test;
 
 public class TestUtilities extends TestCase {
 
@@ -77,9 +73,9 @@ public class TestUtilities extends TestC
     List<ExprNodeDesc> children = new ArrayList<ExprNodeDesc>(1);
     children.add(constant);
     ExprNodeGenericFuncDesc desc = new ExprNodeGenericFuncDesc(TypeInfoFactory.timestampTypeInfo,
-      new GenericUDFFromUtcTimestamp(), children);
+        new GenericUDFFromUtcTimestamp(), children);
     assertEquals(desc.getExprString(), Utilities.deserializeExpression(
-      Utilities.serializeExpression(desc)).getExprString());
+        Utilities.serializeExpression(desc)).getExprString());
   }
 
   public void testgetDbTableName() throws HiveException{
@@ -109,23 +105,4 @@ public class TestUtilities extends TestC
       assertEquals("Invalid table name " + tablename, ex.getMessage());
     }
   }
-
-  @Test
-  public void testFSUmaskReset() throws Exception {
-    // ensure that FS Umask is not reset (HIVE-7001)
-    checkFSUMaskReset(true);
-    checkFSUMaskReset(false);
-  }
-
-  private void checkFSUMaskReset(boolean recursiveArg) throws IllegalArgumentException, IOException {
-    final String FS_MASK_VAL = "055";
-    HiveConf conf = new HiveConf();
-    String dir = System.getProperty("test.tmp.dir") + "/testUtilitiesUMaskReset";
-    conf.set(FsPermission.UMASK_LABEL, FS_MASK_VAL);
-    Utilities.createDirsWithPermission(conf, new Path(dir), new FsPermission((short) 00777),
-        recursiveArg);
-    assertEquals(conf.get(FsPermission.UMASK_LABEL), FS_MASK_VAL);
-  }
-
-
 }

Modified: hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRawRecordMerger.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRawRecordMerger.java?rev=1622597&r1=1622596&r2=1622597&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRawRecordMerger.java (original)
+++ hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRawRecordMerger.java Fri Sep  5 02:32:20 2014
@@ -56,6 +56,7 @@ import java.util.List;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertNull;
 
 public class TestOrcRawRecordMerger {
 
@@ -454,9 +455,16 @@ public class TestOrcRawRecordMerger {
 
   static class MyRow {
     Text col1;
+    RecordIdentifier ROW__ID;
+
     MyRow(String val) {
       col1 = new Text(val);
     }
+
+    MyRow(String val, long rowId, long origTxn, int bucket) {
+      col1 = new Text(val);
+      ROW__ID = new RecordIdentifier(origTxn, bucket, rowId);
+    }
   }
 
   static String getValue(OrcStruct event) {
@@ -533,12 +541,12 @@ public class TestOrcRawRecordMerger {
 
     // write a delta
     ru = of.getRecordUpdater(root, options.writingBase(false)
-        .minimumTransactionId(200).maximumTransactionId(200));
-    ru.update(200, 0, 0, new MyRow("update 1"));
-    ru.update(200, 0, 2, new MyRow("update 2"));
-    ru.update(200, 0, 3, new MyRow("update 3"));
-    ru.delete(200, 0, 7);
-    ru.delete(200, 0, 8);
+        .minimumTransactionId(200).maximumTransactionId(200).recordIdColumn(1));
+    ru.update(200, new MyRow("update 1", 0, 0, BUCKET));
+    ru.update(200, new MyRow("update 2", 2, 0, BUCKET));
+    ru.update(200, new MyRow("update 3", 3, 0, BUCKET));
+    ru.delete(200, new MyRow("", 7, 0, BUCKET));
+    ru.delete(200, new MyRow("", 8, 0, BUCKET));
     ru.close(false);
 
     ValidTxnList txnList = new ValidTxnListImpl("200:");
@@ -607,13 +615,13 @@ public class TestOrcRawRecordMerger {
     assertEquals(OrcRecordUpdater.DELETE_OPERATION,
         OrcRecordUpdater.getOperation(event));
     assertEquals(new ReaderKey(0, BUCKET, 7, 200), id);
-    assertEquals(null, OrcRecordUpdater.getRow(event));
+    assertNull(OrcRecordUpdater.getRow(event));
 
     assertEquals(true, merger.next(id, event));
     assertEquals(OrcRecordUpdater.DELETE_OPERATION,
         OrcRecordUpdater.getOperation(event));
     assertEquals(new ReaderKey(0, BUCKET, 8, 200), id);
-    assertEquals(null, OrcRecordUpdater.getRow(event));
+    assertNull(OrcRecordUpdater.getRow(event));
 
     assertEquals(true, merger.next(id, event));
     assertEquals(OrcRecordUpdater.INSERT_OPERATION,
@@ -693,7 +701,7 @@ public class TestOrcRawRecordMerger {
     assertEquals(OrcRecordUpdater.DELETE_OPERATION,
         OrcRecordUpdater.getOperation(event));
     assertEquals(new ReaderKey(0, BUCKET, 7, 200), id);
-    assertEquals(null, OrcRecordUpdater.getRow(event));
+    assertNull(OrcRecordUpdater.getRow(event));
 
     assertEquals(true, merger.next(id, event));
     assertEquals(OrcRecordUpdater.INSERT_OPERATION,
@@ -705,8 +713,7 @@ public class TestOrcRawRecordMerger {
     assertEquals(OrcRecordUpdater.DELETE_OPERATION,
         OrcRecordUpdater.getOperation(event));
     assertEquals(new ReaderKey(0, BUCKET, 8, 200), id);
-    assertEquals(null, OrcRecordUpdater.getRow(event));
-
+    assertNull(OrcRecordUpdater.getRow(event));
     assertEquals(true, merger.next(id, event));
     assertEquals(OrcRecordUpdater.INSERT_OPERATION,
         OrcRecordUpdater.getOperation(event));
@@ -747,6 +754,7 @@ public class TestOrcRawRecordMerger {
     Text mytext;
     float myfloat;
     double mydouble;
+    RecordIdentifier ROW__ID;
 
     BigRow(int myint, long mylong, String mytext, float myfloat, double mydouble) {
       this.myint = myint;
@@ -754,6 +762,21 @@ public class TestOrcRawRecordMerger {
       this.mytext = new Text(mytext);
       this.myfloat = myfloat;
       this.mydouble = mydouble;
+      ROW__ID = null;
+    }
+
+    BigRow(int myint, long mylong, String mytext, float myfloat, double mydouble,
+                    long rowId, long origTxn, int bucket) {
+      this.myint = myint;
+      this.mylong = mylong;
+      this.mytext = new Text(mytext);
+      this.myfloat = myfloat;
+      this.mydouble = mydouble;
+      ROW__ID = new RecordIdentifier(origTxn, bucket, rowId);
+    }
+
+    BigRow(long rowId, long origTxn, int bucket) {
+      ROW__ID = new RecordIdentifier(origTxn, bucket, rowId);
     }
   }
 
@@ -802,16 +825,16 @@ public class TestOrcRawRecordMerger {
     // write a delta
     AcidOutputFormat.Options options = new AcidOutputFormat.Options(conf)
         .writingBase(false).minimumTransactionId(1).maximumTransactionId(1)
-        .bucket(BUCKET).inspector(inspector).filesystem(fs);
+        .bucket(BUCKET).inspector(inspector).filesystem(fs).recordIdColumn(5);
     RecordUpdater ru = of.getRecordUpdater(root, options);
     values = new String[]{"0.0", null, null, "1.1", null, null, null,
         "ignore.7"};
     for(int i=0; i < values.length; ++i) {
       if (values[i] != null) {
-        ru.update(1, 0, i, new BigRow(i, i, values[i], i, i));
+        ru.update(1, new BigRow(i, i, values[i], i, i, i, 0, BUCKET));
       }
     }
-    ru.delete(100, 0, 9);
+    ru.delete(100, new BigRow(9, 0, BUCKET));
     ru.close(false);
 
     // write a delta
@@ -820,10 +843,10 @@ public class TestOrcRawRecordMerger {
     values = new String[]{null, null, "1.0", null, null, null, null, "3.1"};
     for(int i=0; i < values.length; ++i) {
       if (values[i] != null) {
-        ru.update(2, 0, i, new BigRow(i, i, values[i], i, i));
+        ru.update(2, new BigRow(i, i, values[i], i, i, i, 0, BUCKET));
       }
     }
-    ru.delete(100, 0, 8);
+    ru.delete(100, new BigRow(8, 0, BUCKET));
     ru.close(false);
 
     InputFormat inf = new OrcInputFormat();
@@ -902,16 +925,16 @@ public class TestOrcRawRecordMerger {
     ru.close(false);
 
     // write a delta
-    options.writingBase(false).minimumTransactionId(1).maximumTransactionId(1);
+    options.writingBase(false).minimumTransactionId(1).maximumTransactionId(1).recordIdColumn(5);
     ru = of.getRecordUpdater(root, options);
     values = new String[]{"0.0", null, null, "1.1", null, null, null,
         "ignore.7"};
     for(int i=0; i < values.length; ++i) {
       if (values[i] != null) {
-        ru.update(1, 0, i, new BigRow(i, i, values[i], i, i));
+        ru.update(1, new BigRow(i, i, values[i], i, i, i, 0, BUCKET));
       }
     }
-    ru.delete(100, 0, 9);
+    ru.delete(100, new BigRow(9, 0, BUCKET));
     ru.close(false);
 
     // write a delta
@@ -920,10 +943,10 @@ public class TestOrcRawRecordMerger {
     values = new String[]{null, null, "1.0", null, null, null, null, "3.1"};
     for(int i=0; i < values.length; ++i) {
       if (values[i] != null) {
-        ru.update(2, 0, i, new BigRow(i, i, values[i], i, i));
+        ru.update(2, new BigRow(i, i, values[i], i, i, i, 0, BUCKET));
       }
     }
-    ru.delete(100, 0, 8);
+    ru.delete(100, new BigRow(8, 0, BUCKET));
     ru.close(false);
 
     InputFormat inf = new OrcInputFormat();

Modified: hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRecordUpdater.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRecordUpdater.java?rev=1622597&r1=1622596&r2=1622597&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRecordUpdater.java (original)
+++ hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRecordUpdater.java Fri Sep  5 02:32:20 2014
@@ -23,8 +23,8 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.ql.io.AcidOutputFormat;
 import org.apache.hadoop.hive.ql.io.AcidUtils;
+import org.apache.hadoop.hive.ql.io.RecordIdentifier;
 import org.apache.hadoop.hive.ql.io.RecordUpdater;
-import org.apache.hadoop.hive.serde2.SerDeStats;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
 import org.apache.hadoop.io.IntWritable;
@@ -37,6 +37,7 @@ import java.io.DataInputStream;
 import java.io.File;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
 
 public class TestOrcRecordUpdater {
 
@@ -64,9 +65,18 @@ public class TestOrcRecordUpdater {
 
   static class MyRow {
     Text field;
+    RecordIdentifier ROW__ID;
+
     MyRow(String val) {
       field = new Text(val);
+      ROW__ID = null;
+    }
+
+    MyRow(String val, long rowId, long origTxn, int bucket) {
+      field = new Text(val);
+      ROW__ID = new RecordIdentifier(origTxn, bucket, rowId);
     }
+
   }
 
   @Test
@@ -178,17 +188,19 @@ public class TestOrcRecordUpdater {
       inspector = ObjectInspectorFactory.getReflectionObjectInspector
           (MyRow.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
     }
+    int bucket = 20;
     AcidOutputFormat.Options options = new AcidOutputFormat.Options(conf)
         .filesystem(fs)
-        .bucket(20)
+        .bucket(bucket)
         .writingBase(false)
         .minimumTransactionId(100)
         .maximumTransactionId(100)
         .inspector(inspector)
-        .reporter(Reporter.NULL);
+        .reporter(Reporter.NULL)
+        .recordIdColumn(1);
     RecordUpdater updater = new OrcRecordUpdater(root, options);
-    updater.update(100, 10, 30, new MyRow("update"));
-    updater.delete(100, 40, 60);
+    updater.update(100, new MyRow("update", 30, 10, bucket));
+    updater.delete(100, new MyRow("", 60, 40, bucket));
     assertEquals(-1L, updater.getStats().getRowCount());
     updater.close(false);
     Path bucketPath = AcidUtils.createFilename(root, options);
@@ -216,7 +228,7 @@ public class TestOrcRecordUpdater {
     assertEquals(40, OrcRecordUpdater.getOriginalTransaction(row));
     assertEquals(20, OrcRecordUpdater.getBucket(row));
     assertEquals(60, OrcRecordUpdater.getRowId(row));
-    assertEquals(null, OrcRecordUpdater.getRow(row));
+    assertNull(OrcRecordUpdater.getRow(row));
     assertEquals(false, rows.hasNext());
   }
 }



Mime
View raw message