hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ser...@apache.org
Subject [45/61] [abbrv] hive git commit: HIVE-17706: Add a possibility to run the BeeLine tests on the default database (Peter Vary, reviewed by Barna Zsombor Klara)
Date Tue, 10 Oct 2017 00:52:19 GMT
HIVE-17706: Add a possibility to run the BeeLine tests on the default database (Peter Vary,
reviewed by Barna Zsombor Klara)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1dccdead
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1dccdead
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1dccdead

Branch: refs/heads/hive-14535
Commit: 1dccdeadaba35a0206b17d512ef1cf9aced374b8
Parents: 75c31e7
Author: Peter Vary <pvary@cloudera.com>
Authored: Mon Oct 9 12:09:06 2017 +0200
Committer: Peter Vary <pvary@cloudera.com>
Committed: Mon Oct 9 12:09:06 2017 +0200

----------------------------------------------------------------------
 .../hive/cli/control/CoreBeeLineDriver.java     |   6 +-
 .../java/org/apache/hive/beeline/QFile.java     |  65 ++++++---
 .../apache/hive/beeline/QFileBeeLineClient.java | 138 +++++++++++++++----
 3 files changed, 159 insertions(+), 50 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/1dccdead/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBeeLineDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBeeLineDriver.java
b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBeeLineDriver.java
index b44ffbd..66ccd6f 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBeeLineDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBeeLineDriver.java
@@ -49,7 +49,7 @@ public class CoreBeeLineDriver extends CliAdapter {
   private final File testDataDirectory;
   private final File testScriptDirectory;
   private boolean overwrite = false;
-  private boolean rewriteSourceTables = true;
+  private boolean useSharedDatabase = false;
   private MiniHS2 miniHS2;
   private QFileClientBuilder clientBuilder;
   private QFileBuilder fileBuilder;
@@ -111,7 +111,7 @@ public class CoreBeeLineDriver extends CliAdapter {
   public void beforeClass() throws Exception {
     overwrite = getBooleanPropertyValue("test.output.overwrite", Boolean.FALSE);
 
-    rewriteSourceTables = getBooleanPropertyValue("test.rewrite.source.tables", Boolean.TRUE);
+    useSharedDatabase = getBooleanPropertyValue("test.beeline.shared.database", Boolean.FALSE);
 
     String beeLineUrl = System.getProperty("test.beeline.url");
     if (StringUtils.isEmpty(beeLineUrl)) {
@@ -132,7 +132,7 @@ public class CoreBeeLineDriver extends CliAdapter {
         .setLogDirectory(logDirectory)
         .setQueryDirectory(queryDirectory)
         .setResultsDirectory(resultsDirectory)
-        .setRewriteSourceTables(rewriteSourceTables)
+        .setUseSharedDatabase(useSharedDatabase)
         .setComparePortable(comparePortable);
 
     runInfraScript(initScript, new File(logDirectory, "init.beeline"),

http://git-wip-us.apache.org/repos/asf/hive/blob/1dccdead/itests/util/src/main/java/org/apache/hive/beeline/QFile.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hive/beeline/QFile.java b/itests/util/src/main/java/org/apache/hive/beeline/QFile.java
index 38b0d91..21be8b0 100644
--- a/itests/util/src/main/java/org/apache/hive/beeline/QFile.java
+++ b/itests/util/src/main/java/org/apache/hive/beeline/QFile.java
@@ -81,7 +81,7 @@ public final class QFile {
   private static RegexFilterSet staticFilterSet = getStaticFilterSet();
   private static RegexFilterSet portableFilterSet = getPortableFilterSet();
   private RegexFilterSet specificFilterSet;
-  private boolean rewriteSourceTables;
+  private boolean useSharedDatabase;
   private Converter converter;
   private boolean comparePortable;
 
@@ -91,6 +91,7 @@ public final class QFile {
     return name;
   }
 
+
   public String getDatabaseName() {
     return databaseName;
   }
@@ -127,6 +128,10 @@ public final class QFile {
     return converter;
   }
 
+  public boolean isUseSharedDatabase() {
+    return useSharedDatabase;
+  }
+
   public String getDebugHint() {
     return String.format(DEBUG_HINT, inputFile, rawOutputFile, outputFile, expectedOutputFile,
         logFile, beforeExecuteLogFile, afterExecuteLogFile,
@@ -134,13 +139,13 @@ public final class QFile {
   }
 
   /**
-   * Filters the sql commands if necessary.
+   * Filters the sql commands if necessary - eg. not using the shared database.
    * @param commands The array of the sql commands before filtering
    * @return The filtered array of the sql command strings
    * @throws IOException File read error
    */
   public String[] filterCommands(String[] commands) throws IOException {
-    if (rewriteSourceTables) {
+    if (!useSharedDatabase) {
       for (int i=0; i<commands.length; i++) {
         if (USE_PATTERN.matcher(commands[i]).matches()) {
           System.err.println(String.format(USE_COMMAND_WARNING, inputFile, commands[i]));
@@ -177,8 +182,8 @@ public final class QFile {
    */
   private String revertReplaceTableNames(String source) {
     for (String table : srcTables) {
-      source = source.replaceAll("(?is)(?<!name:?|alias:?)(\\s+)default\\." + table
-          + "([\\s;\\n\\)])", "$1" + table + "$2");
+      source = source.replaceAll("(?is)(?<!name:?|alias:?)(\\s+)default\\.(" + table
+          + ")([\\s;\\n\\),])", "$1$2$3");
     }
     return source;
   }
@@ -199,18 +204,28 @@ public final class QFile {
     return source;
   }
 
+  /**
+   * Filters the generated output file
+   * @throws IOException
+   */
   public void filterOutput() throws IOException {
     String output = FileUtils.readFileToString(rawOutputFile, "UTF-8");
     if (comparePortable) {
       output = portableFilterSet.filter(output);
     }
     output = staticFilterSet.filter(specificFilterSet.filter(output));
-    if (rewriteSourceTables) {
+    if (!useSharedDatabase) {
       output = sortInputOutput(revertReplaceTableNames(output));
     }
     FileUtils.writeStringToFile(outputFile, output);
   }
 
+  /**
+   * Compare the filtered file with the expected golden file
+   * @return The comparison data
+   * @throws IOException If there is a problem accessing the golden or generated file
+   * @throws InterruptedException If there is a problem running the diff command
+   */
   public QTestProcessExecResult compareResults() throws IOException, InterruptedException
{
     if (!expectedOutputFile.exists()) {
       throw new IOException("Expected results file does not exist: " + expectedOutputFile);
@@ -218,6 +233,10 @@ public final class QFile {
     return executeDiff();
   }
 
+  /**
+   * Overwrite the golden file with the generated output
+   * @throws IOException If there is a problem accessing the golden or generated file
+   */
   public void overwriteResults() throws IOException {
     FileUtils.copyFile(outputFile, expectedOutputFile);
   }
@@ -316,8 +335,11 @@ public final class QFile {
         .addFilter(".*/tmp/.*\n", MASK_PATTERN)
         .addFilter(".*file:.*\n", MASK_PATTERN)
         .addFilter(".*file\\..*\n", MASK_PATTERN)
+        .addFilter(".*Location.*\n", MASK_PATTERN)
+        .addFilter(".*LOCATION '.*\n", MASK_PATTERN)
         .addFilter(".*Output:.*/data/files/.*\n", MASK_PATTERN)
         .addFilter(".*CreateTime.*\n", MASK_PATTERN)
+        .addFilter(".*last_modified_.*\n", MASK_PATTERN)
         .addFilter(".*transient_lastDdlTime.*\n", MASK_PATTERN)
         .addFilter(".*lastUpdateTime.*\n", MASK_PATTERN)
         .addFilter(".*lastAccessTime.*\n", MASK_PATTERN)
@@ -349,7 +371,7 @@ public final class QFile {
     private File queryDirectory;
     private File logDirectory;
     private File resultsDirectory;
-    private boolean rewriteSourceTables;
+    private boolean useSharedDatabase;
     private boolean comparePortable;
 
     public QFileBuilder() {
@@ -370,8 +392,8 @@ public final class QFile {
       return this;
     }
 
-    public QFileBuilder setRewriteSourceTables(boolean rewriteSourceTables) {
-      this.rewriteSourceTables = rewriteSourceTables;
+    public QFileBuilder setUseSharedDatabase(boolean useSharedDatabase) {
+      this.useSharedDatabase = useSharedDatabase;
       return this;
     }
 
@@ -383,22 +405,27 @@ public final class QFile {
     public QFile getQFile(String name) throws IOException {
       QFile result = new QFile();
       result.name = name;
-      result.databaseName = "test_db_" + name;
+      if (!useSharedDatabase) {
+        result.databaseName = "test_db_" + name.toLowerCase();
+        result.specificFilterSet = new RegexFilterSet()
+            .addFilter("(PREHOOK|POSTHOOK): (Output|Input): database:" + result.databaseName
+ "\n",
+                "$1: $2: database:default\n")
+            .addFilter("(PREHOOK|POSTHOOK): (Output|Input): " + result.databaseName + "@",
+                "$1: $2: default@")
+            .addFilter("name(:?) " + result.databaseName + "\\.(.*)\n", "name$1 default.$2\n")
+            .addFilter("alias(:?) " + result.databaseName + "\\.(.*)\n", "alias$1 default.$2\n")
+            .addFilter("/" + result.databaseName + ".db/", "/");
+      } else {
+        result.databaseName = "default";
+        result.specificFilterSet = new RegexFilterSet();
+      }
       result.inputFile = new File(queryDirectory, name + ".q");
       result.rawOutputFile = new File(logDirectory, name + ".q.out.raw");
       result.outputFile = new File(logDirectory, name + ".q.out");
       result.logFile = new File(logDirectory, name + ".q.beeline");
       result.beforeExecuteLogFile = new File(logDirectory, name + ".q.beforeExecute.log");
       result.afterExecuteLogFile = new File(logDirectory, name + ".q.afterExecute.log");
-      result.rewriteSourceTables = rewriteSourceTables;
-      result.specificFilterSet = new RegexFilterSet()
-          .addFilter("(PREHOOK|POSTHOOK): (Output|Input): database:" + result.databaseName
+ "\n",
-              "$1: $2: database:default\n")
-          .addFilter("(PREHOOK|POSTHOOK): (Output|Input): " + result.databaseName + "@",
-              "$1: $2: default@")
-          .addFilter("name(:?) " + result.databaseName + "\\.(.*)\n", "name$1 default.$2\n")
-          .addFilter("alias(:?) " + result.databaseName + "\\.(.*)\n", "alias$1 default.$2\n")
-          .addFilter("/" + result.databaseName + ".db/", "/");
+      result.useSharedDatabase = useSharedDatabase;
       result.converter = Converter.NONE;
       String input = FileUtils.readFileToString(result.inputFile, "UTF-8");
       if (input.contains("-- SORT_QUERY_RESULTS")) {

http://git-wip-us.apache.org/repos/asf/hive/blob/1dccdead/itests/util/src/main/java/org/apache/hive/beeline/QFileBeeLineClient.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hive/beeline/QFileBeeLineClient.java b/itests/util/src/main/java/org/apache/hive/beeline/QFileBeeLineClient.java
index 2f91834..d84ff39 100644
--- a/itests/util/src/main/java/org/apache/hive/beeline/QFileBeeLineClient.java
+++ b/itests/util/src/main/java/org/apache/hive/beeline/QFileBeeLineClient.java
@@ -18,12 +18,19 @@
 
 package org.apache.hive.beeline;
 
+import org.apache.commons.lang3.ArrayUtils;
+import org.apache.hadoop.hive.ql.QTestUtil;
 import org.apache.hive.beeline.ConvertedOutputFile.Converter;
 
 import java.io.File;
 import java.io.IOException;
 import java.io.PrintStream;
+import java.sql.DatabaseMetaData;
+import java.sql.ResultSet;
 import java.sql.SQLException;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.stream.Collectors;
 
 /**
  * QFile test client using BeeLine. It can be used to submit a list of command strings, or
a QFile.
@@ -32,6 +39,26 @@ public class QFileBeeLineClient implements AutoCloseable {
   private BeeLine beeLine;
   private PrintStream beelineOutputStream;
   private File logFile;
+  private String[] TEST_FIRST_COMMANDS = new String[] {
+    "!set outputformat tsv2",
+    "!set verbose false",
+    "!set silent true",
+    "!set showheader false",
+    "USE default;",
+    "SHOW TABLES;",
+  };
+  private String[] TEST_SET_LOG_COMMANDS = new String[] {
+    "set hive.testing.short.logs=true;",
+    "set hive.testing.remove.logs=false;",
+  };
+  private String[] TEST_RESET_COMMANDS = new String[] {
+    "set hive.testing.short.logs=false;",
+    "!set verbose true",
+    "!set silent false",
+    "!set showheader true",
+    "!set outputformat table",
+    "USE default;"
+  };
 
   protected QFileBeeLineClient(String jdbcUrl, String jdbcDriver, String username, String
password,
       File log) throws IOException {
@@ -52,6 +79,47 @@ public class QFileBeeLineClient implements AutoCloseable {
         });
   }
 
+  private Set<String> getDatabases() throws SQLException {
+    Set<String> databases = new HashSet<String>();
+
+    DatabaseMetaData metaData = beeLine.getDatabaseMetaData();
+    // Get the databases
+    try (ResultSet schemasResultSet = metaData.getSchemas()) {
+      while (schemasResultSet.next()) {
+        databases.add(schemasResultSet.getString("TABLE_SCHEM"));
+      }
+    }
+    return databases;
+  }
+
+  private Set<String> getTables() throws SQLException {
+    Set<String> tables = new HashSet<String>();
+
+    DatabaseMetaData metaData = beeLine.getDatabaseMetaData();
+    // Get the tables in the default database
+    String[] types = new String[] {"TABLE"};
+    try (ResultSet tablesResultSet = metaData.getTables(null, "default", "%", types)) {
+      while (tablesResultSet.next()) {
+        tables.add(tablesResultSet.getString("TABLE_NAME"));
+      }
+    }
+    return tables;
+  }
+
+  private Set<String> getViews() throws SQLException {
+    Set<String> views = new HashSet<String>();
+
+    DatabaseMetaData metaData = beeLine.getDatabaseMetaData();
+    // Get the tables in the default database
+    String[] types = new String[] {"VIEW"};
+    try (ResultSet tablesResultSet = metaData.getTables(null, "default", "%", types)) {
+      while (tablesResultSet.next()) {
+        views.add(tablesResultSet.getString("TABLE_NAME"));
+      }
+    }
+    return views;
+  }
+
   public void execute(String[] commands, File resultFile, Converter converter)
       throws Exception {
     beeLine.runCommands(
@@ -69,39 +137,53 @@ public class QFileBeeLineClient implements AutoCloseable {
   }
 
   private void beforeExecute(QFile qFile) throws Exception {
-    execute(
-        new String[] {
-          "!set outputformat tsv2",
-          "!set verbose false",
-          "!set silent true",
-          "!set showheader false",
-          "USE default;",
-          "SHOW TABLES;",
-          "DROP DATABASE IF EXISTS `" + qFile.getDatabaseName() + "` CASCADE;",
-          "CREATE DATABASE `" + qFile.getDatabaseName() + "`;",
-          "USE `" + qFile.getDatabaseName() + "`;",
-          "set hive.testing.short.logs=true;",
-          "set hive.testing.remove.logs=false;",
-        },
-        qFile.getBeforeExecuteLogFile(),
-        Converter.NONE);
+    String[] commands = TEST_FIRST_COMMANDS;
+
+    String[] extraCommands;
+    if (qFile.isUseSharedDatabase()) {
+      // If we are using a shared database, then remove not known databases, tables, views.
+      Set<String> dropCommands = getDatabases().stream()
+          .filter(database -> !database.equals("default"))
+          .map(database -> "DROP DATABASE `" + database + "` CASCADE;")
+          .collect(Collectors.toSet());
+
+      Set<String> srcTables = QTestUtil.getSrcTables();
+      dropCommands.addAll(getTables().stream()
+          .filter(table -> !srcTables.contains(table))
+          .map(table -> "DROP TABLE `" + table + "` PURGE;")
+          .collect(Collectors.toSet()));
+
+      dropCommands.addAll(getViews().stream()
+          .map(view -> "DROP VIEW `" + view + "`;")
+          .collect(Collectors.toSet()));
+      extraCommands = dropCommands.toArray(new String[]{});
+    } else {
+      // If we are using a test specific database, then we just drop the database, and recreate
+      extraCommands = new String[] {
+        "DROP DATABASE IF EXISTS `" + qFile.getDatabaseName() + "` CASCADE;",
+        "CREATE DATABASE `" + qFile.getDatabaseName() + "`;",
+        "USE `" + qFile.getDatabaseName() + "`;"
+      };
+    }
+    commands = ArrayUtils.addAll(commands, extraCommands);
+    commands = ArrayUtils.addAll(commands, TEST_SET_LOG_COMMANDS);
+    execute(commands, qFile.getBeforeExecuteLogFile(), Converter.NONE);
     beeLine.setIsTestMode(true);
   }
 
   private void afterExecute(QFile qFile) throws Exception {
     beeLine.setIsTestMode(false);
-    execute(
-        new String[] {
-          "set hive.testing.short.logs=false;",
-          "!set verbose true",
-          "!set silent false",
-          "!set showheader true",
-          "!set outputformat table",
-          "USE default;",
-          "DROP DATABASE IF EXISTS `" + qFile.getDatabaseName() + "` CASCADE;",
-        },
-        qFile.getAfterExecuteLogFile(),
-        Converter.NONE);
+    String[] commands = TEST_RESET_COMMANDS;
+
+    if (!qFile.isUseSharedDatabase()) {
+      // If we are using a test specific database, then we just drop the database
+      String[] extraCommands = new String[] {
+          "DROP DATABASE IF EXISTS `" + qFile.getDatabaseName() + "` CASCADE;"
+      };
+      commands = ArrayUtils.addAll(commands, extraCommands);
+    }
+
+    execute(commands, qFile.getAfterExecuteLogFile(), Converter.NONE);
   }
 
   public void execute(QFile qFile) throws Exception {


Mime
View raw message