Return-Path: 
 <commits-return-16810-apmail-hive-commits-archive=hive.apache.org@hive.apache.org>
X-Original-To: apmail-hive-commits-archive@www.apache.org
Delivered-To: apmail-hive-commits-archive@www.apache.org
Received: from mail.apache.org (hermes.apache.org [140.211.11.3])
	by minotaur.apache.org (Postfix) with SMTP id EC31418B24
	for <apmail-hive-commits-archive@www.apache.org>;
 Tue, 25 Aug 2015 21:28:31 +0000 (UTC)
Received: (qmail 9706 invoked by uid 500); 25 Aug 2015 21:28:31 -0000
Delivered-To: apmail-hive-commits-archive@hive.apache.org
Received: (qmail 9573 invoked by uid 500); 25 Aug 2015 21:28:31 -0000
Mailing-List: contact commits-help@hive.apache.org; run by ezmlm
Precedence: bulk
List-Help: <mailto:commits-help@hive.apache.org>
List-Unsubscribe: <mailto:commits-unsubscribe@hive.apache.org>
List-Post: <mailto:commits@hive.apache.org>
List-Id: <commits.hive.apache.org>
Reply-To: hive-dev@hive.apache.org
Delivered-To: mailing list commits@hive.apache.org
Received: (qmail 9290 invoked by uid 99); 25 Aug 2015 21:28:31 -0000
Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org)
 (140.211.11.23)
    by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 25 Aug 2015 21:28:31 +0000
Received: by git1-us-west.apache.org (ASF Mail Server at
 git1-us-west.apache.org, from userid 33)
	id 3B43DE03E6; Tue, 25 Aug 2015 21:28:31 +0000 (UTC)
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
From: sershe@apache.org
To: commits@hive.apache.org
Date: Tue, 25 Aug 2015 21:28:35 -0000
Message-Id: <6a8b343a185d49fba75c49ab1d5f6c70@git.apache.org>
In-Reply-To: <262b3785a26c4068913ec704c574f42e@git.apache.org>
References: <262b3785a26c4068913ec704c574f42e@git.apache.org>
X-Mailer: ASF-Git Admin Mailer
Subject: [05/50] [abbrv] hive git commit: HIVE-11538 : Add an option to skip
 init script while running tests (Ashutosh Chauhan via Sergey Shelukhin)

HIVE-11538 : Add an option to skip init script while running tests (Ashutosh Chauhan via Sergey Shelukhin)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6e762919
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6e762919
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6e762919

Branch: refs/heads/llap
Commit: 6e7629193ed7b8714bd5ae5ab48bd10c2cbd85cf
Parents: 433ea9c
Author: Ashutosh Chauhan <hashutosh@apache.org>
Authored: Thu Aug 13 20:42:33 2015 -0700
Committer: Ashutosh Chauhan <hashutosh@apache.org>
Committed: Thu Aug 13 20:43:46 2015 -0700

----------------------------------------------------------------------
 itests/qtest/pom.xml                            | 26 ++++----
 .../org/apache/hadoop/hive/ql/QTestUtil.java    | 62 ++++++++++----------
 2 files changed, 44 insertions(+), 44 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/6e762919/itests/qtest/pom.xml
----------------------------------------------------------------------
diff --git a/itests/qtest/pom.xml b/itests/qtest/pom.xml
index 44d30da..0588233 100644
--- a/itests/qtest/pom.xml
+++ b/itests/qtest/pom.xml
@@ -30,7 +30,7 @@
 
   <properties>
     <hive.path.to.root>../..</hive.path.to.root>
-
+    <initScript>q_test_init.sql</initScript>
     <qfile></qfile>
     <qfile_regex></qfile_regex>
     <run_disabled>false</run_disabled>
@@ -420,7 +420,7 @@
                   logFile="${project.build.directory}/testparseneggen.log"
                   hadoopVersion="${active.hadoop.version}"
                   logDirectory="${project.build.directory}/qfile-results/negative/"
-                  initScript="q_test_init.sql"
+                  initScript="${initScript}"
                   cleanupScript="q_test_cleanup.sql"/>
 
                 <!-- Cli -->
@@ -437,7 +437,7 @@
                   logFile="${project.build.directory}/testclidrivergen.log"
                   logDirectory="${project.build.directory}/qfile-results/clientpositive/"
                   hadoopVersion="${active.hadoop.version}"
-                  initScript="q_test_init.sql"
+                  initScript="${initScript}"
                   cleanupScript="q_test_cleanup.sql"/>
 
                 <!-- Negative Cli -->
@@ -454,7 +454,7 @@
                   logFile="${project.build.directory}/testnegativeclidrivergen.log"
                   logDirectory="${project.build.directory}/qfile-results/clientnegative/"
                   hadoopVersion="${active.hadoop.version}"
-                  initScript="q_test_init.sql"
+                  initScript="${initScript}"
                   cleanupScript="q_test_cleanup.sql"/>
 
                 <!-- Compare Cli -->
@@ -470,7 +470,7 @@
                   logFile="${project.build.directory}/testcompareclidrivergen.log"
                   logDirectory="${project.build.directory}/qfile-results/clientcompare/"
                   hadoopVersion="${active.hadoop.version}"
-                  initScript="q_test_init.sql"
+                  initScript="${initScript}"
                   cleanupScript="q_test_cleanup.sql"/>
 
                 <!-- Minimr -->
@@ -487,7 +487,7 @@
                   logFile="${project.build.directory}/testminimrclidrivergen.log"
                   logDirectory="${project.build.directory}/qfile-results/clientpositive/"
                   hadoopVersion="${active.hadoop.version}"
-                  initScript="q_test_init.sql"
+                  initScript="${initScript}"
                   cleanupScript="q_test_cleanup.sql"/>
 
                 <if>
@@ -508,7 +508,7 @@
                               logFile="${project.build.directory}/testminitezclidrivergen.log"
                               logDirectory="${project.build.directory}/qfile-results/clientpositive/"
                               hadoopVersion="${active.hadoop.version}"
-                              initScript="q_test_init.sql"
+                              initScript="${initScript}"
                               cleanupScript="q_test_cleanup.sql"/>
 
                     <qtestgen hiveRootDirectory="${basedir}/${hive.path.to.root}/"
@@ -547,7 +547,7 @@
                   logFile="${project.build.directory}/testnegativeminimrclidrivergen.log"
                   logDirectory="${project.build.directory}/qfile-results/clientnegative/"
                   hadoopVersion="${hadoopVersion}"
-                  initScript="q_test_init.sql"
+                  initScript="${initScript}"
                   cleanupScript="q_test_cleanup.sql"/>
 
                 <!-- HBase Positive -->
@@ -561,7 +561,7 @@
                   resultsDirectory="${basedir}/${hive.path.to.root}/hbase-handler/src/test/results/positive/" className="TestHBaseCliDriver"
                   logFile="${project.build.directory}/testhbaseclidrivergen.log"
                   logDirectory="${project.build.directory}/qfile-results/hbase-handler/positive/"
-                  initScript="q_test_init.sql"
+                  initScript="${initScript}"
                   cleanupScript="q_test_cleanup.sql"/>
 
                 <!-- HBase Minimr -->
@@ -575,7 +575,7 @@
                   resultsDirectory="${basedir}/${hive.path.to.root}/hbase-handler/src/test/results/positive/" className="TestHBaseMinimrCliDriver"
                   logFile="${project.build.directory}/testhbaseminimrclidrivergen.log"
                   logDirectory="${project.build.directory}/qfile-results/hbase-handler/minimrpositive/"
-                  initScript="q_test_init.sql"
+                  initScript="${initScript}"
                   cleanupScript="q_test_cleanup.sql"/>
 
                 <!-- HBase Negative -->
@@ -589,7 +589,7 @@
                   resultsDirectory="${basedir}/${hive.path.to.root}/hbase-handler/src/test/results/negative/" className="TestHBaseNegativeCliDriver"
                   logFile="${project.build.directory}/testhbasenegativeclidrivergen.log"
                   logDirectory="${project.build.directory}/qfile-results/hbase-handler/negative"
-                  initScript="q_test_init.sql"
+                  initScript="${initScript}"
                   cleanupScript="q_test_cleanup.sql"/>
 
                 <!-- Beeline -->
@@ -626,7 +626,7 @@
                   logFile="${project.build.directory}/testcontribclidrivergen.log"
                   logDirectory="${project.build.directory}/qfile-results/contribclientpositive"
                   hadoopVersion="${hadoopVersion}"
-                  initScript="q_test_init.sql"
+                  initScript="${initScript}"
                   cleanupScript="q_test_cleanup.sql"/>
 
                 <qtestgen hiveRootDirectory="${basedir}/${hive.path.to.root}/"
@@ -639,7 +639,7 @@
                   resultsDirectory="${basedir}/${hive.path.to.root}/contrib/src/test/results/clientnegative/" className="TestContribNegativeCliDriver"
                   logFile="${project.build.directory}/testcontribnegclidrivergen.log"
                   logDirectory="${project.build.directory}/qfile-results/contribclientnegative"
-                  initScript="q_test_init.sql"
+                  initScript="${initScript}"
                   cleanupScript="q_test_cleanup.sql"/>
 
 

http://git-wip-us.apache.org/repos/asf/hive/blob/6e762919/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index 39d5d9e..3fae0ba 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -121,8 +121,8 @@ public class QTestUtil {
 
   private static final Log LOG = LogFactory.getLog("QTestUtil");
   private static final String QTEST_LEAVE_FILES = "QTEST_LEAVE_FILES";
-  private final String defaultInitScript = "q_test_init.sql";
-  private final String defaultCleanupScript = "q_test_cleanup.sql";
+  private final static String defaultInitScript = "q_test_init.sql";
+  private final static String defaultCleanupScript = "q_test_cleanup.sql";
   private final String[] testOnlyCommands = new String[]{"crypto"};
 
   private String testWarehouse;
@@ -149,7 +149,6 @@ public class QTestUtil {
   private HadoopShims.MiniMrShim mr = null;
   private HadoopShims.MiniDFSShim dfs = null;
   private HadoopShims.HdfsEncryptionShim hes = null;
-  private boolean miniMr = false;
   private String hadoopVer = null;
   private QTestSetup setup = null;
   private SparkSession sparkSession = null;
@@ -209,7 +208,7 @@ public class QTestUtil {
           continue;
         }
 
-        if (file.isDir()) {
+        if (file.isDirectory()) {
           if (!destFs.exists(local_path)) {
             destFs.mkdirs(local_path);
           }
@@ -410,14 +409,9 @@ public class QTestUtil {
     if (scriptsDir == null) {
       scriptsDir = new File(".").getAbsolutePath() + "/data/scripts";
     }
-    if (initScript.isEmpty()) {
-      initScript = defaultInitScript;
-    }
-    if (cleanupScript.isEmpty()) {
-      cleanupScript = defaultCleanupScript;
-    }
-    this.initScript = scriptsDir + "/" + initScript;
-    this.cleanupScript = scriptsDir + "/" + cleanupScript;
+
+    this.initScript = scriptsDir + File.separator + initScript;
+    this.cleanupScript = scriptsDir + File.separator + cleanupScript;
 
     overWrite = "true".equalsIgnoreCase(System.getProperty("test.output.overwrite"));
 
@@ -705,7 +699,7 @@ public class QTestUtil {
       FileSystem fileSystem = p.getFileSystem(conf);
       if (fileSystem.exists(p)) {
         for (FileStatus status : fileSystem.listStatus(p)) {
-          if (status.isDir() && !srcTables.contains(status.getPath().getName())) {
+          if (status.isDirectory() && !srcTables.contains(status.getPath().getName())) {
             fileSystem.delete(status.getPath(), true);
           }
         }
@@ -755,16 +749,19 @@ public class QTestUtil {
     clearTablesCreatedDuringTests();
     clearKeysCreatedInTests();
 
-    SessionState.get().getConf().setBoolean("hive.test.shutdown.phase", true);
-
-    String cleanupCommands = readEntireFileIntoString(new File(cleanupScript));
-    LOG.info("Cleanup (" + cleanupScript + "):\n" + cleanupCommands);
-    if(cliDriver == null) {
-      cliDriver = new CliDriver();
+    File cleanupFile = new File(cleanupScript);
+    if (cleanupFile.isFile()) {
+      String cleanupCommands = readEntireFileIntoString(cleanupFile);
+      LOG.info("Cleanup (" + cleanupScript + "):\n" + cleanupCommands);
+      if(cliDriver == null) {
+        cliDriver = new CliDriver();
+      }
+      SessionState.get().getConf().setBoolean("hive.test.shutdown.phase", true);
+      cliDriver.processLine(cleanupCommands);
+      SessionState.get().getConf().setBoolean("hive.test.shutdown.phase", false);
+    } else {
+      LOG.info("No cleanup script detected. Skipping.");
     }
-    cliDriver.processLine(cleanupCommands);
-
-    SessionState.get().getConf().setBoolean("hive.test.shutdown.phase", false);
 
     // delete any contents in the warehouse dir
     Path p = new Path(testWarehouse);
@@ -809,14 +806,21 @@ public class QTestUtil {
     if(!isSessionStateStarted) {
       startSessionState();
     }
-    conf.setBoolean("hive.test.init.phase", true);
 
-    String initCommands = readEntireFileIntoString(new File(this.initScript));
-    LOG.info("Initial setup (" + initScript + "):\n" + initCommands);
     if(cliDriver == null) {
       cliDriver = new CliDriver();
     }
     cliDriver.processLine("set test.data.dir=" + testFiles + ";");
+    File scriptFile = new File(this.initScript);
+    if (!scriptFile.isFile()) {
+      LOG.info("No init script detected. Skipping");
+      return;
+    }
+    conf.setBoolean("hive.test.init.phase", true);
+
+    String initCommands = readEntireFileIntoString(scriptFile);
+    LOG.info("Initial setup (" + initScript + "):\n" + initCommands);
+
     cliDriver.processLine(initCommands);
 
     conf.setBoolean("hive.test.init.phase", false);
@@ -912,6 +916,7 @@ public class QTestUtil {
 
   private CliSessionState createSessionState() {
    return new CliSessionState(conf) {
+      @Override
       public void setSparkSession(SparkSession sparkSession) {
         super.setSparkSession(sparkSession);
         if (sparkSession != null) {
@@ -1136,11 +1141,6 @@ public class QTestUtil {
     return commands;
   }
 
-  private boolean isComment(final String line) {
-    String lineTrimmed = line.trim();
-    return lineTrimmed.startsWith("#") || lineTrimmed.startsWith("--");
-  }
-
   public boolean shouldBeSkipped(String tname) {
     return qSkipSet.contains(tname);
   }
@@ -1816,7 +1816,7 @@ public class QTestUtil {
   {
     QTestUtil[] qt = new QTestUtil[qfiles.length];
     for (int i = 0; i < qfiles.length; i++) {
-      qt[i] = new QTestUtil(resDir, logDir, MiniClusterType.none, null, "0.20", "", "");
+      qt[i] = new QTestUtil(resDir, logDir, MiniClusterType.none, null, "0.20", defaultInitScript, defaultCleanupScript);
       qt[i].addFile(qfiles[i]);
       qt[i].clearTestSideEffects();
     }