hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From na...@apache.org
Subject svn commit: r1095287 - in /hive/trunk: common/src/java/org/apache/hadoop/hive/conf/ conf/ ql/src/java/org/apache/hadoop/hive/ql/ ql/src/java/org/apache/hadoop/hive/ql/exec/ ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/ ql/src/java/org/apache/h...
Date Wed, 20 Apr 2011 06:29:56 GMT
Author: namit
Date: Wed Apr 20 06:29:56 2011
New Revision: 1095287

URL: http://svn.apache.org/viewvc?rev=1095287&view=rev
Log:
HIVE-2109 No lock for some non-mapred tasks config variable
hive.lock.mapred.only.operation added (HE Yongqiang via namit)


Added:
    hive/trunk/ql/src/test/queries/clientpositive/lock4.q
    hive/trunk/ql/src/test/results/clientpositive/lock4.q.out
Modified:
    hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
    hive/trunk/conf/hive-default.xml
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapredLocalTask.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/BlockMergeTask.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/DDLWork.java

Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1095287&r1=1095286&r2=1095287&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Wed Apr 20 06:29:56
2011
@@ -385,6 +385,7 @@ public class HiveConf extends Configurat
     HIVE_LOCK_MANAGER("hive.lock.manager", "org.apache.hadoop.hive.ql.lockmgr.zookeeper.ZooKeeperHiveLockManager"),
     HIVE_LOCK_NUMRETRIES("hive.lock.numretries", 100),
     HIVE_LOCK_SLEEP_BETWEEN_RETRIES("hive.lock.sleep.between.retries", 60),
+    HIVE_LOCK_MAPRED_ONLY("hive.lock.mapred.only.operation", false),
 
     HIVE_ZOOKEEPER_QUORUM("hive.zookeeper.quorum", ""),
     HIVE_ZOOKEEPER_CLIENT_PORT("hive.zookeeper.client.port", ""),
@@ -427,7 +428,7 @@ public class HiveConf extends Configurat
     // temporary variable for testing. This is added just to turn off this feature in case
of a bug in
     // deployment. It has not been documented in hive-default.xml intentionally, this should
be removed
     // once the feature is stable
-    HIVE_MAPPER_CANNOT_SPAN_MULTIPLE_PARTITIONS("hive.mapper.cannot.span.multiple.partitions",
false),
+    HIVE_MAPPER_CANNOT_SPAN_MULTIPLE_PARTITIONS("hive.mapper.cannot.span.multiple.partitions",
false), 
     ;
 
 

Modified: hive/trunk/conf/hive-default.xml
URL: http://svn.apache.org/viewvc/hive/trunk/conf/hive-default.xml?rev=1095287&r1=1095286&r2=1095287&view=diff
==============================================================================
--- hive/trunk/conf/hive-default.xml (original)
+++ hive/trunk/conf/hive-default.xml Wed Apr 20 06:29:56 2011
@@ -999,6 +999,13 @@
 </property>
 
 <property>
+  <name>hive.lock.mapred.only.operation</name>
+  <value>false</value>
+  <description>This param is to control whether or not only do lock on queries 
+  that need to execute at least one mapred job.</description>
+</property>
+
+<property>
   <name>hive.limit.row.max.size</name>
   <value>100000</value>
   <description>When trying a smaller subset of data for simple LIMIT, how much size
we need to guarantee

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java?rev=1095287&r1=1095286&r2=1095287&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java Wed Apr 20 06:29:56 2011
@@ -129,10 +129,12 @@ public class Driver implements CommandPr
   private final int sleeptime = 2000;
   protected int tryCount = Integer.MAX_VALUE;
 
-
-  private int checkLockManager() {
+  private boolean checkLockManager() {
     boolean supportConcurrency = conf.getBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY);
-    if (supportConcurrency && (hiveLockMgr == null)) {
+    if (!supportConcurrency) {
+      return false;
+    }
+    if ((hiveLockMgr == null)) {
       try {
         setLockManager();
       } catch (SemanticException e) {
@@ -140,10 +142,10 @@ public class Driver implements CommandPr
         SQLState = ErrorMsg.findSQLState(e.getMessage());
         console.printError(errorMessage, "\n"
             + org.apache.hadoop.util.StringUtils.stringifyException(e));
-        return (12);
+        return false;
       }
     }
-    return (0);
+    return true;
   }
 
   private void setLockManager() throws SemanticException {
@@ -719,11 +721,6 @@ public class Driver implements CommandPr
         return 0;
       }
 
-      int ret = checkLockManager();
-      if (ret != 0) {
-        return ret;
-      }
-
       HiveLockObjectData lockData =
         new HiveLockObjectData(plan.getQueryId(),
                                String.valueOf(System.currentTimeMillis()),
@@ -739,7 +736,6 @@ public class Driver implements CommandPr
         throw new SemanticException(e.getMessage());
       }
 
-      ctx.setHiveLockMgr(hiveLockMgr);
       List<HiveLock> hiveLocks = null;
 
       int tryNum = 1;
@@ -819,23 +815,56 @@ public class Driver implements CommandPr
 
     int ret = compile(command);
 
-    if (ret != 0) {
-      releaseLocks(ctx.getHiveLocks());
-      return new CommandProcessorResponse(ret, errorMessage, SQLState);
-    }
+    boolean requireLock = false;
+    boolean ckLock = checkLockManager();
 
-    ret = acquireReadWriteLocks();
     if (ret != 0) {
       releaseLocks(ctx.getHiveLocks());
       return new CommandProcessorResponse(ret, errorMessage, SQLState);
     }
+    
+    if (ckLock) {
+      ctx.setHiveLockMgr(hiveLockMgr);
+      boolean lockOnlyMapred = HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_LOCK_MAPRED_ONLY);
+      if(lockOnlyMapred) {
+        Queue<Task<? extends Serializable>> taskQueue = new LinkedList<Task<?
extends Serializable>>();
+        taskQueue.addAll(plan.getRootTasks());
+        while (taskQueue.peek() != null) {
+          Task<? extends Serializable> tsk = taskQueue.remove();
+          requireLock = requireLock || tsk.requireLock();
+          if(requireLock) {
+            break;
+          }
+          if (tsk instanceof ConditionalTask) {
+            taskQueue.addAll(((ConditionalTask)tsk).getListTasks());
+          }
+          if(tsk.getChildTasks()!= null) {
+            taskQueue.addAll(tsk.getChildTasks());        
+          }
+          // does not add back up task here, because back up task should be the same
+          // type of the original task. 
+        }
+      } else {
+        requireLock = true;
+      }
+    }
+    
+    if (requireLock) {
+      ret = acquireReadWriteLocks();
+      if (ret != 0) {
+        releaseLocks(ctx.getHiveLocks());
+        return new CommandProcessorResponse(ret, errorMessage, SQLState);
+      }
+    }
 
     ret = execute();
     if (ret != 0) {
+      //if needRequireLock is false, the release here will do nothing because there is no
lock
       releaseLocks(ctx.getHiveLocks());
       return new CommandProcessorResponse(ret, errorMessage, SQLState);
     }
 
+    //if needRequireLock is false, the release here will do nothing because there is no lock
     releaseLocks(ctx.getHiveLocks());
     return new CommandProcessorResponse(ret);
   }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java?rev=1095287&r1=1095286&r2=1095287&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java Wed Apr 20 06:29:56
2011
@@ -163,6 +163,10 @@ public class DDLTask extends Task<DDLWor
   private static String INTERMEDIATE_ORIGINAL_DIR_SUFFIX;
   private static String INTERMEDIATE_EXTRACTED_DIR_SUFFIX;
 
+  public boolean requireLock() {
+    return this.work != null && this.work.getNeedLock();
+  }
+
   public DDLTask() {
     super();
   }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java?rev=1095287&r1=1095286&r2=1095287&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java Wed Apr 20 06:29:56
2011
@@ -106,6 +106,10 @@ public class ExecDriver extends Task<Map
     this.jobExecHelper = new HadoopJobExecHelper(job, console, this, this);
   }
 
+  public boolean requireLock() {
+    return true;
+  }
+
   protected static String getResourceFiles(Configuration conf, SessionState.ResourceType
t) {
     // fill in local files to be added to the task environment
     SessionState ss = SessionState.get();

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapredLocalTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapredLocalTask.java?rev=1095287&r1=1095286&r2=1095287&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapredLocalTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapredLocalTask.java Wed Apr 20
06:29:56 2011
@@ -97,7 +97,9 @@ public class MapredLocalTask extends Tas
     return sdf.format(cal.getTime());
   }
 
-
+  public boolean requireLock() {
+    return true;
+  }
 
   @Override
   public int execute(DriverContext driverContext) {

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java?rev=1095287&r1=1095286&r2=1095287&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java Wed Apr 20 06:29:56 2011
@@ -474,6 +474,10 @@ public abstract class Task<T extends Ser
   public void setLocalMode(boolean isLocalMode) {
     this.isLocalMode = isLocalMode;
   }
+  
+  public boolean requireLock() {
+    return false;
+  }
 
   public boolean ifRetryCmdWhenFail() {
     return retryCmdWhenFail;

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/BlockMergeTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/BlockMergeTask.java?rev=1095287&r1=1095286&r2=1095287&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/BlockMergeTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/BlockMergeTask.java Wed
Apr 20 06:29:56 2011
@@ -74,6 +74,10 @@ public class BlockMergeTask extends Task
     job = new JobConf(conf, BlockMergeTask.class);
     jobExecHelper = new HadoopJobExecHelper(job, this.console, this, this);
   }
+  
+  public boolean requireLock() {
+    return true;
+  }
 
   boolean success = true;
 

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java?rev=1095287&r1=1095286&r2=1095287&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java Wed Apr
20 06:29:56 2011
@@ -1174,8 +1174,9 @@ public class DDLSemanticAnalyzer extends
     mergeDesc.setOutputDir(outputDir);
 
     addInputsOutputsAlterTable(tableName, partSpec);
-    Task<? extends Serializable> mergeTask = TaskFactory.get(new DDLWork(
-        getInputs(), getOutputs(), mergeDesc), conf);
+    DDLWork ddlWork = new DDLWork(getInputs(), getOutputs(), mergeDesc);
+    ddlWork.setNeedLock(true);
+    Task<? extends Serializable> mergeTask = TaskFactory.get(ddlWork, conf);
 
     tableSpec tablepart = new tableSpec(this.db, conf, tablePartAST);
     StatsWork statDesc = new StatsWork(tablepart);

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/DDLWork.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/DDLWork.java?rev=1095287&r1=1095286&r2=1095287&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/DDLWork.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/DDLWork.java Wed Apr 20 06:29:56
2011
@@ -66,6 +66,8 @@ public class DDLWork implements Serializ
   private RevokeDesc revokeDesc;
   private GrantRevokeRoleDDL grantRevokeRoleDDL;
 
+  boolean needLock = false;
+  
   /**
    * ReadEntitites that are passed to the hooks.
    */
@@ -877,4 +879,12 @@ public class DDLWork implements Serializ
     this.mergeFilesDesc = mergeDesc;
   }
 
+  public boolean getNeedLock() {
+    return needLock;
+  }
+
+  public void setNeedLock(boolean needLock) {
+    this.needLock = needLock;
+  }
+
 }

Added: hive/trunk/ql/src/test/queries/clientpositive/lock4.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/lock4.q?rev=1095287&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/lock4.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/lock4.q Wed Apr 20 06:29:56 2011
@@ -0,0 +1,33 @@
+set hive.lock.mapred.only.operation=true;
+drop table tstsrcpart;
+create table tstsrcpart like srcpart;
+
+from srcpart
+insert overwrite table tstsrcpart partition (ds='2008-04-08',hr='11')
+select key, value where ds='2008-04-08' and hr='11';
+
+set hive.exec.dynamic.partition.mode=nonstrict;
+set hive.exec.dynamic.partition=true;
+
+
+from srcpart
+insert overwrite table tstsrcpart partition (ds, hr) select key, value, ds, hr where ds <=
'2008-04-08';
+
+from srcpart
+insert overwrite table tstsrcpart partition (ds ='2008-04-08', hr) select key, value, hr
where ds = '2008-04-08';
+
+
+SHOW LOCKS;
+SHOW LOCKS tstsrcpart;
+
+drop table tstsrcpart;
+
+drop table tst1;
+create table tst1 (key string, value string) partitioned by (a string, b string, c string,
d string);
+
+
+from srcpart
+insert overwrite table tst1 partition (a='1', b='2', c, d) select key, value, ds, hr where
ds = '2008-04-08';
+
+
+drop table tst1;

Added: hive/trunk/ql/src/test/results/clientpositive/lock4.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/lock4.q.out?rev=1095287&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/lock4.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/lock4.q.out Wed Apr 20 06:29:56 2011
@@ -0,0 +1,189 @@
+PREHOOK: query: drop table tstsrcpart
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table tstsrcpart
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table tstsrcpart like srcpart
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: create table tstsrcpart like srcpart
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@tstsrcpart
+PREHOOK: query: from srcpart
+insert overwrite table tstsrcpart partition (ds='2008-04-08',hr='11')
+select key, value where ds='2008-04-08' and hr='11'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11
+POSTHOOK: query: from srcpart
+insert overwrite table tstsrcpart partition (ds='2008-04-08',hr='11')
+select key, value where ds='2008-04-08' and hr='11'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value,
type:string, comment:default), ]
+PREHOOK: query: from srcpart
+insert overwrite table tstsrcpart partition (ds, hr) select key, value, ds, hr where ds <=
'2008-04-08'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+PREHOOK: Output: default@tstsrcpart
+POSTHOOK: query: from srcpart
+insert overwrite table tstsrcpart partition (ds, hr) select key, value, ds, hr where ds <=
'2008-04-08'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11
+POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value,
type:string, comment:default), ]
+PREHOOK: query: from srcpart
+insert overwrite table tstsrcpart partition (ds ='2008-04-08', hr) select key, value, hr
where ds = '2008-04-08'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+PREHOOK: Output: default@tstsrcpart@ds=2008-04-08
+POSTHOOK: query: from srcpart
+insert overwrite table tstsrcpart partition (ds ='2008-04-08', hr) select key, value, hr
where ds = '2008-04-08'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11
+POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value,
type:string, comment:default), ]
+PREHOOK: query: SHOW LOCKS
+PREHOOK: type: SHOWLOCKS
+POSTHOOK: query: SHOW LOCKS
+POSTHOOK: type: SHOWLOCKS
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value,
type:string, comment:default), ]
+PREHOOK: query: SHOW LOCKS tstsrcpart
+PREHOOK: type: SHOWLOCKS
+POSTHOOK: query: SHOW LOCKS tstsrcpart
+POSTHOOK: type: SHOWLOCKS
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value,
type:string, comment:default), ]
+PREHOOK: query: drop table tstsrcpart
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@tstsrcpart
+PREHOOK: Output: default@tstsrcpart
+POSTHOOK: query: drop table tstsrcpart
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@tstsrcpart
+POSTHOOK: Output: default@tstsrcpart
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value,
type:string, comment:default), ]
+PREHOOK: query: drop table tst1
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table tst1
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value,
type:string, comment:default), ]
+PREHOOK: query: create table tst1 (key string, value string) partitioned by (a string, b
string, c string, d string)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: create table tst1 (key string, value string) partitioned by (a string, b
string, c string, d string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@tst1
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value,
type:string, comment:default), ]
+PREHOOK: query: from srcpart
+insert overwrite table tst1 partition (a='1', b='2', c, d) select key, value, ds, hr where
ds = '2008-04-08'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+PREHOOK: Output: default@tst1@a=1/b=2
+POSTHOOK: query: from srcpart
+insert overwrite table tst1 partition (a='1', b='2', c, d) select key, value, ds, hr where
ds = '2008-04-08'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+POSTHOOK: Output: default@tst1@a=1/b=2/c=2008-04-08/d=11
+POSTHOOK: Output: default@tst1@a=1/b=2/c=2008-04-08/d=12
+POSTHOOK: Lineage: tst1 PARTITION(a=1,b=2,c=2008-04-08,d=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key,
type:string, comment:default), ]
+POSTHOOK: Lineage: tst1 PARTITION(a=1,b=2,c=2008-04-08,d=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value,
type:string, comment:default), ]
+POSTHOOK: Lineage: tst1 PARTITION(a=1,b=2,c=2008-04-08,d=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key,
type:string, comment:default), ]
+POSTHOOK: Lineage: tst1 PARTITION(a=1,b=2,c=2008-04-08,d=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value,
type:string, comment:default), ]
+PREHOOK: query: drop table tst1
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@tst1
+PREHOOK: Output: default@tst1
+POSTHOOK: query: drop table tst1
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@tst1
+POSTHOOK: Output: default@tst1
+POSTHOOK: Lineage: tst1 PARTITION(a=1,b=2,c=2008-04-08,d=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key,
type:string, comment:default), ]
+POSTHOOK: Lineage: tst1 PARTITION(a=1,b=2,c=2008-04-08,d=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value,
type:string, comment:default), ]
+POSTHOOK: Lineage: tst1 PARTITION(a=1,b=2,c=2008-04-08,d=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key,
type:string, comment:default), ]
+POSTHOOK: Lineage: tst1 PARTITION(a=1,b=2,c=2008-04-08,d=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key,
type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value,
type:string, comment:default), ]



Mime
View raw message