hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From the...@apache.org
Subject svn commit: r1529047 - in /hive/branches/branch-0.12: beeline/src/java/org/apache/hive/beeline/ beeline/src/test/org/apache/hive/beeline/src/test/ metastore/scripts/upgrade/oracle/ metastore/scripts/upgrade/postgres/ metastore/src/test/org/apache/hadoo...
Date Fri, 04 Oct 2013 00:56:49 GMT
Author: thejas
Date: Fri Oct  4 00:56:49 2013
New Revision: 1529047

URL: http://svn.apache.org/r1529047
Log:
HIVE-5419 : Fix schema tool issues with Oracle metastore (Prasad Mujumdar via Ashutosh Chauhan)

Modified:
    hive/branches/branch-0.12/beeline/src/java/org/apache/hive/beeline/HiveSchemaHelper.java
    hive/branches/branch-0.12/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java
    hive/branches/branch-0.12/beeline/src/test/org/apache/hive/beeline/src/test/TestSchemaTool.java
    hive/branches/branch-0.12/metastore/scripts/upgrade/oracle/014-HIVE-3764.oracle.sql
    hive/branches/branch-0.12/metastore/scripts/upgrade/oracle/hive-schema-0.12.0.oracle.sql
    hive/branches/branch-0.12/metastore/scripts/upgrade/oracle/upgrade-0.11.0-to-0.12.0.oracle.sql
    hive/branches/branch-0.12/metastore/scripts/upgrade/postgres/hive-schema-0.12.0.postgres.sql
    hive/branches/branch-0.12/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java

Modified: hive/branches/branch-0.12/beeline/src/java/org/apache/hive/beeline/HiveSchemaHelper.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/beeline/src/java/org/apache/hive/beeline/HiveSchemaHelper.java?rev=1529047&r1=1529046&r2=1529047&view=diff
==============================================================================
--- hive/branches/branch-0.12/beeline/src/java/org/apache/hive/beeline/HiveSchemaHelper.java
(original)
+++ hive/branches/branch-0.12/beeline/src/java/org/apache/hive/beeline/HiveSchemaHelper.java
Fri Oct  4 00:56:49 2013
@@ -22,7 +22,7 @@ import java.util.IllegalFormatException;
 public class HiveSchemaHelper {
   public static final String DB_DERBY = "derby";
   public static final String DB_MYSQL = "mysql";
-  public static final String DB_POSTGRACE = "postgrace";
+  public static final String DB_POSTGRACE = "postgres";
   public static final String DB_ORACLE = "oracle";
 
   public interface NestedScriptParser {
@@ -225,8 +225,8 @@ public class HiveSchemaHelper {
       if (!isNestedScript(dbCommand)) {
         throw new IllegalArgumentException("Not a nested script format " + dbCommand);
       }
-      // remove ending ';'
-      return dbCommand.replace(";", "");
+      // remove ending ';' and starting '@'
+      return dbCommand.replace(";", "").replace(ORACLE_NESTING_TOKEN, "");
     }
 
     @Override

Modified: hive/branches/branch-0.12/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java?rev=1529047&r1=1529046&r2=1529047&view=diff
==============================================================================
--- hive/branches/branch-0.12/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java
(original)
+++ hive/branches/branch-0.12/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java
Fri Oct  4 00:56:49 2013
@@ -366,6 +366,7 @@ public class HiveSchemaTool {
       beeLine.getOpts().setSilent(true);
     }
     beeLine.getOpts().setAllowMultiLineCommand(false);
+    beeLine.getOpts().setIsolation("TRANSACTION_READ_COMMITTED");
     int status = beeLine.begin(argList.toArray(new String[0]), null);
     if (status != 0) {
       throw new IOException("Schema script failed, errorcode " + status);

Modified: hive/branches/branch-0.12/beeline/src/test/org/apache/hive/beeline/src/test/TestSchemaTool.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/beeline/src/test/org/apache/hive/beeline/src/test/TestSchemaTool.java?rev=1529047&r1=1529046&r2=1529047&view=diff
==============================================================================
--- hive/branches/branch-0.12/beeline/src/test/org/apache/hive/beeline/src/test/TestSchemaTool.java
(original)
+++ hive/branches/branch-0.12/beeline/src/test/org/apache/hive/beeline/src/test/TestSchemaTool.java
Fri Oct  4 00:56:49 2013
@@ -30,6 +30,7 @@ import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.HiveMetaException;
+import org.apache.hadoop.hive.metastore.MetaStoreSchemaInfo;
 import org.apache.hive.beeline.HiveSchemaHelper;
 import org.apache.hive.beeline.HiveSchemaHelper.NestedScriptParser;
 import org.apache.hive.beeline.HiveSchemaTool;
@@ -100,7 +101,8 @@ public class TestSchemaTool extends Test
    * @throws Exception
    */
   public void testSchemaInit() throws Exception {
-    schemaTool.doInit("0.12.0");
+    schemaTool.doInit(MetaStoreSchemaInfo.getHiveSchemaVersion());
+    schemaTool.verifySchemaVersion();
     }
 
   /**
@@ -321,6 +323,51 @@ public class TestSchemaTool extends Test
     assertEquals(expectedSQL, flattenedSql);
   }
 
+  /**
+   * Test nested script formatting
+   * @throws Exception
+   */
+  public void testNestedScriptsForOracle() throws Exception {
+    String childTab1 = "childTab1";
+    String childTab2 = "childTab2";
+    String parentTab = "fooTab";
+
+    String childTestScript1[] = {
+      "-- this is a comment ",
+      "DROP TABLE IF EXISTS " + childTab1 + ";",
+      "CREATE TABLE " + childTab1 + "(id INTEGER);",
+      "DROP TABLE " + childTab1 + ";"
+    };
+    String childTestScript2[] = {
+        "-- this is a comment",
+        "DROP TABLE IF EXISTS " + childTab2 + ";",
+        "CREATE TABLE " + childTab2 + "(id INTEGER);",
+        "-- this is also a comment",
+        "DROP TABLE " + childTab2 + ";"
+    };
+
+    String parentTestScript[] = {
+        " -- this is a comment",
+        "DROP TABLE IF EXISTS " + parentTab + ";",
+        " -- this is another comment ",
+        "CREATE TABLE " + parentTab + "(id INTEGER);",
+        "@" + generateTestScript(childTestScript1).getName() + ";",
+        "DROP TABLE " + parentTab + ";",
+        "@" + generateTestScript(childTestScript2).getName() + ";",
+        "--ending comment ",
+      };
+
+    File testScriptFile = generateTestScript(parentTestScript);
+    String flattenedSql = HiveSchemaTool.buildCommand(
+        HiveSchemaHelper.getDbCommandParser("oracle"),
+        testScriptFile.getParentFile().getPath(), testScriptFile.getName());
+    assertFalse(flattenedSql.contains("@"));
+    assertFalse(flattenedSql.contains("comment"));
+    assertTrue(flattenedSql.contains(childTab1));
+    assertTrue(flattenedSql.contains(childTab2));
+    assertTrue(flattenedSql.contains(parentTab));
+  }
+
   private File generateTestScript(String [] stmts) throws IOException {
     File testScriptFile = File.createTempFile("schematest", ".sql");
     testScriptFile.deleteOnExit();

Modified: hive/branches/branch-0.12/metastore/scripts/upgrade/oracle/014-HIVE-3764.oracle.sql
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/metastore/scripts/upgrade/oracle/014-HIVE-3764.oracle.sql?rev=1529047&r1=1529046&r2=1529047&view=diff
==============================================================================
--- hive/branches/branch-0.12/metastore/scripts/upgrade/oracle/014-HIVE-3764.oracle.sql (original)
+++ hive/branches/branch-0.12/metastore/scripts/upgrade/oracle/014-HIVE-3764.oracle.sql Fri
Oct  4 00:56:49 2013
@@ -1,10 +1,10 @@
 -- HIVE-3764 Support metastore version consistency check
 
-CREATE TABLE IF NOT EXISTS VERSION (
+CREATE TABLE VERSION (
   VER_ID NUMBER NOT NULL,
   SCHEMA_VERSION VARCHAR(127) NOT NULL,
   VERSION_COMMENT VARCHAR(255)
-)
+);
 ALTER TABLE VERSION ADD CONSTRAINT VERSION_PK PRIMARY KEY (VER_ID);
 
-INSERT INTO VERSION (VER_ID, SCHEMA_VERSION, VERSION_COMMENT) VALUES (1, '', 'Initial value');
+INSERT INTO VERSION (VER_ID, SCHEMA_VERSION, VERSION_COMMENT) VALUES (1, ' ', 'Initial value');

Modified: hive/branches/branch-0.12/metastore/scripts/upgrade/oracle/hive-schema-0.12.0.oracle.sql
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/metastore/scripts/upgrade/oracle/hive-schema-0.12.0.oracle.sql?rev=1529047&r1=1529046&r2=1529047&view=diff
==============================================================================
--- hive/branches/branch-0.12/metastore/scripts/upgrade/oracle/hive-schema-0.12.0.oracle.sql
(original)
+++ hive/branches/branch-0.12/metastore/scripts/upgrade/oracle/hive-schema-0.12.0.oracle.sql
Fri Oct  4 00:56:49 2013
@@ -483,11 +483,11 @@ CREATE TABLE TAB_COL_STATS (
  LAST_ANALYZED NUMBER NOT NULL
 );
 
-CREATE TABLE IF NOT EXISTS VERSION (
+CREATE TABLE VERSION (
   VER_ID NUMBER NOT NULL,
   SCHEMA_VERSION VARCHAR(127) NOT NULL,
   VERSION_COMMENT VARCHAR(255)
-)
+);
 ALTER TABLE VERSION ADD CONSTRAINT VERSION_PK PRIMARY KEY (VER_ID);
 
 ALTER TABLE TAB_COL_STATS ADD CONSTRAINT TAB_COL_STATS_PKEY PRIMARY KEY (CS_ID);

Modified: hive/branches/branch-0.12/metastore/scripts/upgrade/oracle/upgrade-0.11.0-to-0.12.0.oracle.sql
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/metastore/scripts/upgrade/oracle/upgrade-0.11.0-to-0.12.0.oracle.sql?rev=1529047&r1=1529046&r2=1529047&view=diff
==============================================================================
--- hive/branches/branch-0.12/metastore/scripts/upgrade/oracle/upgrade-0.11.0-to-0.12.0.oracle.sql
(original)
+++ hive/branches/branch-0.12/metastore/scripts/upgrade/oracle/upgrade-0.11.0-to-0.12.0.oracle.sql
Fri Oct  4 00:56:49 2013
@@ -1,4 +1,4 @@
-SELECT 'Upgrading MetaStore schema from 0.11.0 to 0.12.0' AS ' ';
+SELECT 'Upgrading MetaStore schema from 0.11.0 to 0.12.0' AS Status from dual;
 @013-HIVE-3255.oracle.sql;
 @014-HIVE-3764.oracle.sql;
 UPDATE VERSION SET SCHEMA_VERSION='0.12.0', VERSION_COMMENT='Hive release version 0.12.0'
where VER_ID=1;

Modified: hive/branches/branch-0.12/metastore/scripts/upgrade/postgres/hive-schema-0.12.0.postgres.sql
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/metastore/scripts/upgrade/postgres/hive-schema-0.12.0.postgres.sql?rev=1529047&r1=1529046&r2=1529047&view=diff
==============================================================================
--- hive/branches/branch-0.12/metastore/scripts/upgrade/postgres/hive-schema-0.12.0.postgres.sql
(original)
+++ hive/branches/branch-0.12/metastore/scripts/upgrade/postgres/hive-schema-0.12.0.postgres.sql
Fri Oct  4 00:56:49 2013
@@ -521,7 +521,7 @@ CREATE TABLE "TAB_COL_STATS" (
 CREATE TABLE "VERSION" (
   "VER_ID" bigint,
   "SCHEMA_VERSION" character varying(127) NOT NULL,
-  "VERSION_COMMENT" character varying(255) NOT NULL,
+  "VERSION_COMMENT" character varying(255) NOT NULL
 );
 
 --

Modified: hive/branches/branch-0.12/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java?rev=1529047&r1=1529046&r2=1529047&view=diff
==============================================================================
--- hive/branches/branch-0.12/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java
(original)
+++ hive/branches/branch-0.12/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java
Fri Oct  4 00:56:49 2013
@@ -17,8 +17,13 @@
  */
 package org.apache.hadoop.hive.metastore;
 
+import java.io.File;
+import java.lang.reflect.Field;
+import java.util.Random;
+
 import junit.framework.TestCase;
 
+import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.hive.cli.CliSessionState;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.api.MetaException;
@@ -31,18 +36,35 @@ public class TestMetastoreVersion extend
   protected HiveConf hiveConf;
   private Driver driver;
   private String hiveHome;
+  private String testMetastoreDB;
+  Random randomNum = new Random();
 
   @Override
   protected void setUp() throws Exception {
     super.setUp();
+    Field defDb = HiveMetaStore.HMSHandler.class.getDeclaredField("createDefaultDB");
+    defDb.setAccessible(true);
+    defDb.setBoolean(null, false);
     hiveConf = new HiveConf(this.getClass());
     System.setProperty("hive.metastore.event.listeners",
         DummyListener.class.getName());
     System.setProperty("hive.metastore.pre.event.listeners",
         DummyPreListener.class.getName());
+    testMetastoreDB = System.getProperty("java.io.tmpdir") +
+    File.separator + "test_metastore-" + randomNum.nextInt();
+    System.setProperty(HiveConf.ConfVars.METASTORECONNECTURLKEY.varname,
+        "jdbc:derby:" + testMetastoreDB + ";create=true");
     hiveHome = System.getProperty("hive.home");
   }
 
+  @Override
+  protected void tearDown() throws Exception {
+    File metaStoreDir = new File(testMetastoreDB);
+    if (metaStoreDir.exists()) {
+      FileUtils.deleteDirectory(metaStoreDir);
+    }
+  }
+
   /***
    * Test config defaults
    */



Mime
View raw message