hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From gunt...@apache.org
Subject svn commit: r1661816 - in /hive/branches/llap: itests/util/src/main/java/org/apache/hadoop/hive/ql/ llap-client/src/java/org/apache/hadoop/hive/llap/io/api/ llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/ ql/src/java/org/apache/hadoop/hiv...
Date Mon, 23 Feb 2015 22:55:28 GMT
Author: gunther
Date: Mon Feb 23 22:55:27 2015
New Revision: 1661816

URL: http://svn.apache.org/r1661816
Log:
HIVE-9654: LLAP: initialize IO during service startup, with service configuration (Gunther
Hagleitner)

Modified:
    hive/branches/llap/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
    hive/branches/llap/llap-client/src/java/org/apache/hadoop/hive/llap/io/api/LlapIoProxy.java
    hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
    hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java

Modified: hive/branches/llap/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java?rev=1661816&r1=1661815&r2=1661816&view=diff
==============================================================================
--- hive/branches/llap/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
(original)
+++ hive/branches/llap/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
Mon Feb 23 22:55:27 2015
@@ -73,6 +73,7 @@ import org.apache.hadoop.hive.common.io.
 import org.apache.hadoop.hive.common.io.SortPrintStream;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.llap.io.api.LlapIoProxy;
 import org.apache.hadoop.hive.metastore.MetaStoreUtils;
 import org.apache.hadoop.hive.metastore.api.Index;
 import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
@@ -80,6 +81,7 @@ import org.apache.hadoop.hive.ql.exec.Ta
 import org.apache.hadoop.hive.ql.exec.Utilities;
 import org.apache.hadoop.hive.ql.exec.spark.session.SparkSession;
 import org.apache.hadoop.hive.ql.exec.spark.session.SparkSessionManagerImpl;
+import org.apache.hadoop.hive.llap.io.api.LlapIoProxy;
 import org.apache.hadoop.hive.ql.lockmgr.zookeeper.CuratorFrameworkSingleton;
 import org.apache.hadoop.hive.ql.lockmgr.zookeeper.ZooKeeperHiveLockManager;
 import org.apache.hadoop.hive.ql.metadata.Hive;
@@ -411,6 +413,9 @@ public class QTestUtil {
 
     initConf();
 
+    LOG.info("initializing llap");
+    LlapIoProxy.initializeLlapIo(conf);
+
     // Use the current directory if it is not specified
     String dataDir = conf.get("test.data.files");
     if (dataDir == null) {

Modified: hive/branches/llap/llap-client/src/java/org/apache/hadoop/hive/llap/io/api/LlapIoProxy.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/llap-client/src/java/org/apache/hadoop/hive/llap/io/api/LlapIoProxy.java?rev=1661816&r1=1661815&r2=1661816&view=diff
==============================================================================
--- hive/branches/llap/llap-client/src/java/org/apache/hadoop/hive/llap/io/api/LlapIoProxy.java
(original)
+++ hive/branches/llap/llap-client/src/java/org/apache/hadoop/hive/llap/io/api/LlapIoProxy.java
Mon Feb 23 22:55:27 2015
@@ -29,23 +29,18 @@ public class LlapIoProxy {
 
   // Llap server depends on Hive execution, so the reverse cannot be true. We create the
I/O
   // singleton once (on daemon startup); the said singleton server as the IO interface.
-  private static final Object instanceLock = new Object();
   private static LlapIo io = null;
 
-
-  // TODO: temporary interface. LLAP daemon should init this once during
-  //       startup then others should use get w/o configuration.
-  public static LlapIo getOrCreateIo(Configuration conf) {
-    if (io != null) return io;
-    synchronized (instanceLock) {
-      if (io != null) return io;
-      initializeLlapIo(conf);
-      return io;
-    }
+  public static LlapIo getIo() {
+    return io;
   }
 
   public static void initializeLlapIo(Configuration conf) {
-    assert io == null;
+
+    if (io != null) {
+      return; // already initialized
+    }
+
     try {
       io = createIoImpl(conf);
     } catch (IOException e) {
@@ -53,10 +48,6 @@ public class LlapIoProxy {
     }
   }
 
-  public static LlapIo getIo() {
-    return io;
-  }
-
   private static LlapIo createIoImpl(Configuration conf) throws IOException {
     try {
       @SuppressWarnings("unchecked")

Modified: hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java?rev=1661816&r1=1661815&r2=1661816&view=diff
==============================================================================
--- hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
(original)
+++ hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
Mon Feb 23 22:55:27 2015
@@ -26,6 +26,7 @@ import org.apache.log4j.Logger;
 import org.apache.hadoop.hive.llap.daemon.ContainerRunner;
 import org.apache.hadoop.hive.llap.daemon.LlapDaemonConfiguration;
 import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.RunContainerRequestProto;
+import org.apache.hadoop.hive.llap.io.api.LlapIoProxy;
 import org.apache.hadoop.hive.llap.shufflehandler.ShuffleHandler;
 
 public class LlapDaemon extends AbstractService implements ContainerRunner {
@@ -84,6 +85,7 @@ public class LlapDaemon extends Abstract
   public void serviceInit(Configuration conf) {
     server.init(conf);
     containerRunner.init(conf);
+    LlapIoProxy.initializeLlapIo(conf);
   }
 
   @Override

Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java?rev=1661816&r1=1661815&r2=1661816&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java Mon Feb
23 22:55:27 2015
@@ -83,7 +83,7 @@ public class HiveInputFormat<K extends W
   /**
    * A cache of InputFormat instances.
    */
-  private static Map<Class, InputFormat<WritableComparable, Writable>> inputFormats

+  private static Map<Class, InputFormat<WritableComparable, Writable>> inputFormats
     = new ConcurrentHashMap<Class, InputFormat<WritableComparable, Writable>>();
 
   private JobConf job;
@@ -213,8 +213,7 @@ public class HiveInputFormat<K extends W
     }
     LOG.info("Wrapping " + inputFormat);
     @SuppressWarnings("unchecked")
-    // TODO: should be LlapIoProxy.getIo eventually
-    LlapIo<VectorizedRowBatch> llapIo = LlapIoProxy.getOrCreateIo(conf);
+    LlapIo<VectorizedRowBatch> llapIo = LlapIoProxy.getIo();
     return castInputFormat(llapIo.getInputFormat(inputFormat));
   }
 



Mime
View raw message