hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ss...@apache.org
Subject hive git commit: Revert "HIVE-13496. Create initial test data once across multiple test runs - TestCliDriver. (Siddharth Seth, reviewed by Ashutosh Chauhan)"
Date Thu, 14 Apr 2016 18:32:32 GMT
Repository: hive
Updated Branches:
  refs/heads/master 940fc7dfe -> e3e43c6dd


Revert "HIVE-13496. Create initial test data once across multiple test runs - TestCliDriver.
(Siddharth Seth, reviewed by Ashutosh Chauhan)"

This reverts commit 976e628fc01911936caa19e61ea3342f3a19455a.


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e3e43c6d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e3e43c6d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e3e43c6d

Branch: refs/heads/master
Commit: e3e43c6dd89b9643d3f7a8f21c4789af24bba6eb
Parents: 940fc7d
Author: Siddharth Seth <sseth@apache.org>
Authored: Thu Apr 14 11:32:19 2016 -0700
Committer: Siddharth Seth <sseth@apache.org>
Committed: Thu Apr 14 11:32:19 2016 -0700

----------------------------------------------------------------------
 .../org/apache/hadoop/hive/ql/QTestUtil.java    | 247 ++-----------------
 pom.xml                                         |   2 -
 ql/src/test/templates/TestCliDriver.vm          |  38 +--
 3 files changed, 22 insertions(+), 265 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/e3e43c6d/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index 79646cd..2f109ab 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -51,6 +51,7 @@ import java.util.Collection;
 import java.util.Comparator;
 import java.util.Deque;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
@@ -62,7 +63,6 @@ import java.util.concurrent.TimeUnit;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-import com.google.common.base.Preconditions;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang.StringUtils;
@@ -84,7 +84,9 @@ import org.apache.hadoop.hive.common.io.SortPrintStream;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.hive.llap.LlapItUtils;
+import org.apache.hadoop.hive.llap.configuration.LlapDaemonConfiguration;
 import org.apache.hadoop.hive.llap.daemon.MiniLlapCluster;
+import org.apache.hadoop.hive.llap.daemon.impl.LlapDaemon;
 import org.apache.hadoop.hive.llap.io.api.LlapProxy;
 import org.apache.hadoop.hive.metastore.MetaStoreUtils;
 import org.apache.hadoop.hive.metastore.api.Index;
@@ -97,7 +99,6 @@ import org.apache.hadoop.hive.ql.exec.tez.TezSessionState;
 import org.apache.hadoop.hive.ql.lockmgr.zookeeper.CuratorFrameworkSingleton;
 import org.apache.hadoop.hive.ql.lockmgr.zookeeper.ZooKeeperHiveLockManager;
 import org.apache.hadoop.hive.ql.metadata.Hive;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.ql.metadata.InvalidTableException;
 import org.apache.hadoop.hive.ql.metadata.Table;
 import org.apache.hadoop.hive.ql.parse.ASTNode;
@@ -138,8 +139,6 @@ public class QTestUtil {
   // security property names
   private static final String SECURITY_KEY_PROVIDER_URI_NAME = "dfs.encryption.key.provider.uri";
   private static final String CRLF = System.getProperty("line.separator");
-  private static final String TEST_BUILD_DIR = System.getProperty("test.build.dir");
-  private static final String CACHED_DATA_DIR_NAME = "cachedData";
 
   private static final Logger LOG = LoggerFactory.getLogger("QTestUtil");
   private static final String QTEST_LEAVE_FILES = "QTEST_LEAVE_FILES";
@@ -184,16 +183,6 @@ public class QTestUtil {
   private final String cleanupScript;
   private boolean useHBaseMetastore = false;
 
-  // Parameters which help tracking cached data generation.
-  private final String driverName;
-  private Path cachedDataPath;
-  private String metaStorePathString;
-  private Path metaStorePath;
-  private FileSystem localFs;
-  private boolean attemptingCacheUsage;
-
-  private boolean dbEtcSetup = false;
-
   public interface SuiteAddTestFunctor {
     public void addTestToSuite(TestSuite suite, Object setup, String tName);
   }
@@ -389,34 +378,11 @@ public class QTestUtil {
   }
 
   public QTestUtil(String outDir, String logDir, MiniClusterType clusterType,
-                   String confDir, String hadoopVer, String initScript, String cleanupScript,
-                   boolean useHBaseMetastore, boolean withLlapIo) throws Exception {
-    // For now, to avoid changing multiple test templates, a null driver name avoids
-    // data generation optimizations.
-    this(outDir, logDir, clusterType, confDir, hadoopVer, initScript, cleanupScript,
-        useHBaseMetastore, withLlapIo, null);
-  }
-
-  public QTestUtil(String outDir, String logDir, MiniClusterType clusterType,
       String confDir, String hadoopVer, String initScript, String cleanupScript,
-      boolean useHBaseMetastore, boolean withLlapIo, String driverName)
+      boolean useHBaseMetastore, boolean withLlapIo)
     throws Exception {
-    this.attemptingCacheUsage = (StringUtils.isEmpty(TEST_BUILD_DIR) ||
-        StringUtils.isEmpty(driverName) || useHBaseMetastore) ? false : true;
-    this.driverName = driverName;
     this.outDir = outDir;
     this.logDir = logDir;
-    LOG.info("Creating QTestUtil with settings: "
-        + "driverName=" + driverName
-        + ", attemptingCacheUsage=" + attemptingCacheUsage
-        + ", test.build.dir=" + System.getProperty("test.build.dir")
-        + ", useHbaseMetaStore=" + useHBaseMetastore
-        + ", withLlapIo=" + withLlapIo
-        + ", confDir=" + confDir
-        + ", outDir=" + outDir
-        + ", logDir=" + logDir
-        + ", initScript=" + initScript
-        + ", cleanupScript=" + cleanupScript);
     this.useHBaseMetastore = useHBaseMetastore;
 
     if (confDir != null && !confDir.isEmpty()) {
@@ -505,7 +471,6 @@ public class QTestUtil {
     if (scriptsDir == null) {
       scriptsDir = new File(".").getAbsolutePath() + "/data/scripts";
     }
-    LOG.info("Using DataDir=" + dataDir + ", ScriptsDir=" + scriptsDir);
 
     this.initScript = scriptsDir + File.separator + initScript;
     this.cleanupScript = scriptsDir + File.separator + cleanupScript;
@@ -867,17 +832,6 @@ public class QTestUtil {
       return;
     }
 
-    if (!attemptingCacheUsage) {
-      cleanupNonCacheUsage();
-    } else {
-      cleanupCacheUsage();
-    }
-
-    FunctionRegistry.unregisterTemporaryUDF("test_udaf");
-    FunctionRegistry.unregisterTemporaryUDF("test_error");
-  }
-
-  private void cleanupNonCacheUsage() throws Exception {
     clearTablesCreatedDuringTests();
     clearKeysCreatedInTests();
 
@@ -895,42 +849,21 @@ public class QTestUtil {
       LOG.info("No cleanup script detected. Skipping.");
     }
 
-    cleanupWarehouseDir();
-  }
-
-  private void cleanupCacheUsage() throws IOException {
-    // Remove the Warehouse and metastore directories completely.
-    // Also close the current db, since files are going to come in to replace it soon.
-    Preconditions.checkState(attemptingCacheUsage);
-    Preconditions.checkNotNull(metaStorePath);
-    Preconditions.checkNotNull(localFs);
-    Hive.closeCurrent();
-    cleanupMetastoreDir();
-    cleanupWarehouseDir();
-  }
-
-  private void cleanupWarehouseDir() throws IOException {
     // delete any contents in the warehouse dir
     Path p = new Path(testWarehouse);
     FileSystem fs = p.getFileSystem(conf);
 
     try {
-      FileStatus[] ls = fs.listStatus(p);
-      for (int i = 0; (ls != null) && (i < ls.length); i++) {
+      FileStatus [] ls = fs.listStatus(p);
+      for (int i=0; (ls != null) && (i<ls.length); i++) {
         fs.delete(ls[i].getPath(), true);
       }
     } catch (FileNotFoundException e) {
       // Best effort
     }
-  }
 
-  private void cleanupMetastoreDir() throws IOException {
-    try {
-      LOG.info("Cleaning up metastore Dir: {}", metaStorePath);
-      localFs.delete(metaStorePath, true);
-    } catch (FileNotFoundException e) {
-      // Best effort
-    }
+    FunctionRegistry.unregisterTemporaryUDF("test_udaf");
+    FunctionRegistry.unregisterTemporaryUDF("test_error");
   }
 
   protected void runCreateTableCmd(String createTableCmd) throws Exception {
@@ -960,10 +893,6 @@ public class QTestUtil {
   }
 
   public void createSources(String tname) throws Exception {
-    createSources(tname, false);
-  }
-
-  public void createSources(String tname, boolean forceCreate) throws Exception {
     boolean canReuseSession = (tname == null) || !qNoSessionReuseQuerySet.contains(tname);
     if(!isSessionStateStarted) {
       startSessionState(canReuseSession);
@@ -972,173 +901,34 @@ public class QTestUtil {
     if(cliDriver == null) {
       cliDriver = new CliDriver();
     }
-
+    cliDriver.processLine("set test.data.dir=" + testFiles + ";");
     File scriptFile = new File(this.initScript);
     if (!scriptFile.isFile()) {
       LOG.info("No init script detected. Skipping");
-      if (attemptingCacheUsage) {
-        setupDbsEtc(true, true);
-      }
       return;
     }
+    conf.setBoolean("hive.test.init.phase", true);
 
-    if (!attemptingCacheUsage || forceCreate) {
-      LOG.info("Creating sources without data caching. attemptingCacheUsage={}, forceCreate={}",
-          attemptingCacheUsage, forceCreate);
-      cliDriver.processLine("set test.data.dir=" + testFiles + ";");
-      conf.setBoolean("hive.test.init.phase", true);
-      createSourcesNonCached(scriptFile);
-    } else {
-      LOG.info("Creating sources with data caching");
-      createSourcesCached(scriptFile);
-    }
-
-    conf.setBoolean("hive.test.init.phase", false);
-  }
-
-  private void createSourcesNonCached(File scriptFile) throws IOException {
     String initCommands = readEntireFileIntoString(scriptFile);
     LOG.info("Initial setup (" + initScript + "):\n" + initCommands);
 
     cliDriver.processLine(initCommands);
-  }
-
-  private void createSourcesCached(File scriptFile) throws IOException, HiveException {
 
-    // First check if the cache already exists. If it does just copy it over.
-    Path cachedWarehousePath = new Path(cachedDataPath, "warehouse");
-    Path cachedMetaStorePtah = new Path(cachedDataPath, "metastore");
-    if (localFs.exists(cachedDataPath)) {
-      if (localFs.exists(cachedWarehousePath) && localFs.exists(cachedMetaStorePtah))
{
-        LOG.info("Cached data found in {}. Attempting to use it", cachedDataPath);
-        // Data is alredy cached
-        // Copy the files over to where they should be
-        Path warehousePath = new Path(testWarehouse);
-        FileSystem warehouseFs = warehousePath.getFileSystem(conf);
-        try {
-          warehouseFs.delete(warehousePath, false);
-        } catch (FileNotFoundException e) {
-          // Does not matter if it does not exist.
-        }
-        warehouseFs.copyFromLocalFile(false, cachedWarehousePath, warehousePath);
-
-        try {
-          localFs.delete(metaStorePath, false);
-        } catch (IOException e) {
-          // Does not matter if it does not exist.
-        }
-        localFs.copyFromLocalFile(false, cachedMetaStorePtah, metaStorePath);
-        setupDbsEtc(true, false);
-        cliDriver.processLine("set test.data.dir=" + testFiles + ";");
-        conf.setBoolean("hive.test.init.phase", true);
-
-        return;
-      } else {
-        // Something is missing. Cleanup. Re-generate and cache
-        LOG.info("Partial or no cached data found at {}. Cache will be created", cachedDataPath);
-        localFs.delete(cachedDataPath, true);
-      }
-    } else {
-      LOG.info("No cached data found at {}. Cache will be created", cachedDataPath);
-      // No caching. Re-generate the data and cache it.
-    }
-
-    // Generate and cache the data
-    setupDbsEtc(true, true);
-    cliDriver.processLine("set test.data.dir=" + testFiles + ";");
-    conf.setBoolean("hive.test.init.phase", true);
-    createSourcesNonCached(scriptFile);
-
-    // Close the DB so that contents can be copied out safely.
-    Hive.closeCurrent();
-
-    // Cache the sources
-    localFs.mkdirs(cachedDataPath);
-
-    Path warehousePath = new Path(testWarehouse);
-    FileSystem warehouseFs = warehousePath.getFileSystem(conf);
-
-    warehouseFs.copyToLocalFile(false, warehousePath, cachedWarehousePath, true);
-    localFs.copyToLocalFile(false, metaStorePath, cachedMetaStorePtah, true);
-
-    // Re-open the DB etc.
-    setupDbsEtc(true, false);
-  }
-
-  private static final Pattern metaStoreUriPattern =
-      Pattern.compile("derby.*?databaseName=(.*?)(;|$)");
-
-  private String getDerbyDbPath(String jdbcConnectString) {
-    if (StringUtils.isEmpty(jdbcConnectString)) {
-      return null;
-    }
-    Matcher matcher = metaStoreUriPattern.matcher(jdbcConnectString);
-    if (matcher.find()) {
-      return matcher.group(1);
-    } else {
-      return null;
-    }
+    conf.setBoolean("hive.test.init.phase", false);
   }
 
   public void init() throws Exception {
-    LOG.info("init");
-    testWarehouse = conf.getVar(HiveConf.ConfVars.METASTOREWAREHOUSE);
-    LOG.info("TestWarehouseDir set to: [{}]", testWarehouse);
-    if (attemptingCacheUsage) {
-      // The derby path comes from METASTORECONNECTURLKEY. Default ends up being target/junit_metastore_db
-      String metaStoreConnectUrl = conf.getVar(ConfVars.METASTORECONNECTURLKEY);
-      LOG.info("MetastoreConnectUrl: " + metaStoreConnectUrl);
-      metaStorePathString = getDerbyDbPath(metaStoreConnectUrl);
-
-      if (metaStorePathString == null) {
-        LOG.warn(
-            "Disabling attempted cache usage since metastore path cannot be determined from
{}",
-            metaStoreConnectUrl);
-        attemptingCacheUsage = false;
-      } else {
-        LOG.info("Metastore url path: " + metaStorePathString);
-        metaStorePath = new Path(metaStorePathString);
-        if (metaStorePath.isAbsolute() && metaStorePathString.split(File.separator).length
>= 3) {
-          // Turn this on only if the path is absolute, and is at least 3 deep - since we'll
be deleting files later.
-          localFs = FileSystem.getLocal(conf).getRaw();
-          assert(TEST_BUILD_DIR != null);
-          cachedDataPath = new Path(TEST_BUILD_DIR, CACHED_DATA_DIR_NAME);
-          cachedDataPath = new Path(cachedDataPath, driverName);
-          LOG.info("Using cachedDataPath: " + cachedDataPath);
-        } else {
-          LOG.warn(
-              "Disableing attempted cache usage since metastore path may not be absolute,
or depth is < 3. MetaStorePath={}",
-              metaStorePathString);
-          metaStorePath = null;
-          attemptingCacheUsage = false;
-        }
 
-      }
-    }
+    testWarehouse = conf.getVar(HiveConf.ConfVars.METASTOREWAREHOUSE);
     String execEngine = conf.get("hive.execution.engine");
     conf.set("hive.execution.engine", "mr");
     SessionState.start(conf);
     conf.set("hive.execution.engine", execEngine);
-
-    if (!attemptingCacheUsage) {
-      setupDbsEtc(true, true);
-    }
-  }
-
-  private void setupDbsEtc(boolean force, boolean isNewDb) throws HiveException {
-    if (!dbEtcSetup || force) {
-      if (isNewDb) {
-        db = Hive.get(conf);
-      } else {
-        db = Hive.getWithFastCheck(conf, false);
-      }
-      LOG.info("Obtained db");
-      drv = new Driver(conf);
-      drv.init();
-      pd = new ParseDriver();
-      sem = new SemanticAnalyzer(conf);
-      dbEtcSetup = true;
-    }
+    db = Hive.get(conf);
+    drv = new Driver(conf);
+    drv.init();
+    pd = new ParseDriver();
+    sem = new SemanticAnalyzer(conf);
   }
 
   public void init(String tname) throws Exception {
@@ -1154,9 +944,8 @@ public class QTestUtil {
   public String cliInit(String tname, boolean recreate) throws Exception {
     if (recreate) {
       cleanUp(tname);
-      createSources(tname, true);
+      createSources(tname);
     }
-    setupDbsEtc(false, true);
 
     HiveConf.setVar(conf, HiveConf.ConfVars.HIVE_AUTHENTICATOR_MANAGER,
     "org.apache.hadoop.hive.ql.security.DummyAuthenticator");

http://git-wip-us.apache.org/repos/asf/hive/blob/e3e43c6d/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 08ef998..77cfaeb 100644
--- a/pom.xml
+++ b/pom.xml
@@ -75,7 +75,6 @@
     <test.hive.hadoop.classpath>${maven.test.classpath}</test.hive.hadoop.classpath>
     <test.log4j.scheme>file://</test.log4j.scheme>
     <test.tmp.dir>${project.build.directory}/tmp</test.tmp.dir>
-    <test.build.dir>${project.build.directory}</test.build.dir>
     <test.tmp.dir.uri>file://${test.tmp.dir}</test.tmp.dir.uri>
     <test.warehouse.dir>${project.build.directory}/warehouse</test.warehouse.dir>
     <test.warehouse.scheme>pfile://</test.warehouse.scheme>
@@ -1027,7 +1026,6 @@
             <test.data.dir>${basedir}/${hive.path.to.root}/data/files</test.data.dir>
             <test.tmp.dir>${test.tmp.dir}</test.tmp.dir>
             <test.tmp.dir.uri>${test.tmp.dir.uri}</test.tmp.dir.uri>
-            <test.build.dir>${test.build.dir}</test.build.dir>
             <test.dfs.mkdir>${test.dfs.mkdir}</test.dfs.mkdir>
             <test.output.overwrite>${test.output.overwrite}</test.output.overwrite>
             <test.warehouse.dir>${test.warehouse.scheme}${test.warehouse.dir}</test.warehouse.dir>

http://git-wip-us.apache.org/repos/asf/hive/blob/e3e43c6d/ql/src/test/templates/TestCliDriver.vm
----------------------------------------------------------------------
diff --git a/ql/src/test/templates/TestCliDriver.vm b/ql/src/test/templates/TestCliDriver.vm
index 1961c75..72cfab9 100644
--- a/ql/src/test/templates/TestCliDriver.vm
+++ b/ql/src/test/templates/TestCliDriver.vm
@@ -17,34 +17,23 @@
  */
 package org.apache.hadoop.hive.cli;
 
-import com.google.common.base.Stopwatch;
 import org.apache.hadoop.hive.ql.QTestUtil;
 import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
-import java.util.concurrent.TimeUnit;
-
 public class $className {
 
-  private static final Logger LOG = LoggerFactory.getLogger(${className}.class);
-
   private static final String HIVE_ROOT = QTestUtil.ensurePathEndsInSlash(System.getProperty("hive.root"));
   private static QTestUtil qt;
 
   static {
 
-    Stopwatch stopwatch = new Stopwatch().start();
-    String message = "Starting TestCliDriver run at " + System.currentTimeMillis();
-    LOG.info(message);
-    System.err.println(message);
     MiniClusterType miniMR = MiniClusterType.valueForString("$clusterMode");
     String hiveConfDir = "$hiveConfDir";
     String initScript = "$initScript";
@@ -55,29 +44,15 @@ public class $className {
       if (!hiveConfDir.isEmpty()) {
         hiveConfDir = HIVE_ROOT + hiveConfDir;
       }
-      // TODO Is ZK startup required for TestCliDriver
-      // TODO Is LlapIo enabled required for TestCliDriver
       qt = new QTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR,
-      hiveConfDir, hadoopVer, initScript, cleanupScript, useHBaseMetastore, true, "$className");
-      message = "QTestUtil instance created. ElapsedTimeSinceStart=" + stopwatch.elapsed(
-          TimeUnit.MILLISECONDS);
-      LOG.info(message);
-      System.err.println(message);
+      hiveConfDir, hadoopVer, initScript, cleanupScript, useHBaseMetastore, true);
 
       // do a one time initialization
       qt.cleanUp();
-      message = "Initialization cleanup done. ElapsedTimeSinceStart=" + stopwatch.elapsed(TimeUnit.MILLISECONDS);
-      LOG.info(message);
-      System.err.println(message);
-
       qt.createSources();
-      message = "Initialization createSources done. ElapsedTimeSinceStart=" + stopwatch.elapsed(TimeUnit.MILLISECONDS);
-      LOG.info(message);
-      System.err.println(message);
 
     } catch (Exception e) {
-      System.err.println("Exception: " + e.getMessage() + ". ElapsedTimeSinceStart="
-          + stopwatch.elapsed(TimeUnit.MILLISECONDS));
+      System.err.println("Exception: " + e.getMessage());
       e.printStackTrace();
       System.err.flush();
       fail("Unexpected exception in static initialization: "+e.getMessage());
@@ -87,7 +62,6 @@ public class $className {
   @Before
   public void setUp() {
     try {
-      // TODO This restarts ZK for each test. Is that requried ?
       qt.clearTestSideEffects();
     } catch (Exception e) {
       System.err.println("Exception: " + e.getMessage());
@@ -139,9 +113,7 @@ public class $className {
   private void runTest(String tname, String fname, String fpath) throws Exception {
     long startTime = System.currentTimeMillis();
     try {
-      String message = "Begin query: " + fname + ", startTime=" + startTime;
-      System.err.println(message);
-      LOG.info(message);
+      System.err.println("Begin query: " + fname);
 
       qt.addFile(fpath);
 
@@ -164,9 +136,7 @@ public class $className {
     }
 
     long elapsedTime = System.currentTimeMillis() - startTime;
-    String message = "Done query: " + fname + " elapsedTime=" + elapsedTime/1000 + "s";
-    System.err.println(message);
-    LOG.info(message);
+    System.err.println("Done query: " + fname + " elapsedTime=" + elapsedTime/1000 + "s");
     assertTrue("Test passed", true);
   }
 }


Mime
View raw message