hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From harisan...@apache.org
Subject hive git commit: HIVE-11428: Adding missing files for Performance: Struct IN() clauses are extremely slow (Hari Sankar Sivarama Subramaniyan, reviewed by Gopal V)
Date Fri, 07 Aug 2015 21:24:28 GMT
Repository: hive
Updated Branches:
  refs/heads/branch-1 0f944edc9 -> 8d6f974ab


HIVE-11428: Adding missing files for Performance: Struct IN() clauses are extremely slow (Hari
Sankar Sivarama Subramaniyan, reviewed by Gopal V)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8d6f974a
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8d6f974a
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8d6f974a

Branch: refs/heads/branch-1
Commit: 8d6f974ab13ca5f4ba30db335102eff3a57ae0ac
Parents: 0f944ed
Author: Hari Subramaniyan <harisankar@apache.org>
Authored: Fri Aug 7 14:23:30 2015 -0700
Committer: Hari Subramaniyan <harisankar@apache.org>
Committed: Fri Aug 7 14:24:16 2015 -0700

----------------------------------------------------------------------
 ql/src/test/queries/clientpositive/structin.q   | 17 +++++
 .../test/results/clientpositive/structin.q.out  | 66 ++++++++++++++++++++
 .../StandardConstantStructObjectInspector.java  | 51 +++++++++++++++
 3 files changed, 134 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/8d6f974a/ql/src/test/queries/clientpositive/structin.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/structin.q b/ql/src/test/queries/clientpositive/structin.q
new file mode 100644
index 0000000..48b31f3
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/structin.q
@@ -0,0 +1,17 @@
+create table t11 (`id` string, `lineid` string);
+set hive.cbo.enable=false;
+set hive.tez.dynamic.partition.pruning=false;
+set hive.vectorized.execution.enabled=true;
+
+explain select * from t11 where struct(`id`, `lineid`)
+IN (
+struct('1234-1111-0074578664','3'),
+struct('1234-1111-0074578695','1'),
+struct('1234-1111-0074580704','1'),
+struct('1234-1111-0074581619','2'),
+struct('1234-1111-0074582745','1'),
+struct('1234-1111-0074586625','1'),
+struct('1234-1111-0074019112','1'),
+struct('1234-1111-0074019610','1'),
+struct('1234-1111-0074022106','1')
+);

http://git-wip-us.apache.org/repos/asf/hive/blob/8d6f974a/ql/src/test/results/clientpositive/structin.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/structin.q.out b/ql/src/test/results/clientpositive/structin.q.out
new file mode 100644
index 0000000..e36fceb
--- /dev/null
+++ b/ql/src/test/results/clientpositive/structin.q.out
@@ -0,0 +1,66 @@
+PREHOOK: query: create table t11 (`id` string, `lineid` string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@t11
+POSTHOOK: query: create table t11 (`id` string, `lineid` string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@t11
+PREHOOK: query: explain select * from t11 where struct(`id`, `lineid`)
+IN (
+struct('1234-1111-0074578664','3'),
+struct('1234-1111-0074578695','1'),
+struct('1234-1111-0074580704','1'),
+struct('1234-1111-0074581619','2'),
+struct('1234-1111-0074582745','1'),
+struct('1234-1111-0074586625','1'),
+struct('1234-1111-0074019112','1'),
+struct('1234-1111-0074019610','1'),
+struct('1234-1111-0074022106','1')
+)
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select * from t11 where struct(`id`, `lineid`)
+IN (
+struct('1234-1111-0074578664','3'),
+struct('1234-1111-0074578695','1'),
+struct('1234-1111-0074580704','1'),
+struct('1234-1111-0074581619','2'),
+struct('1234-1111-0074582745','1'),
+struct('1234-1111-0074586625','1'),
+struct('1234-1111-0074019112','1'),
+struct('1234-1111-0074019610','1'),
+struct('1234-1111-0074022106','1')
+)
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+  Stage: Stage-1
+    Map Reduce
+      Map Operator Tree:
+          TableScan
+            alias: t11
+            Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+            Filter Operator
+              predicate: (struct(id,lineid)) IN (struct('1234-1111-0074578664','3'), struct('1234-1111-0074578695','1'),
struct('1234-1111-0074580704','1'), struct('1234-1111-0074581619','2'), struct('1234-1111-0074582745','1'),
struct('1234-1111-0074586625','1'), struct('1234-1111-0074019112','1'), struct('1234-1111-0074019610','1'),
struct('1234-1111-0074022106','1')) (type: boolean)
+              Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+              Select Operator
+                expressions: id (type: string), lineid (type: string)
+                outputColumnNames: _col0, _col1
+                Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+                File Output Operator
+                  compressed: false
+                  Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats:
NONE
+                  table:
+                      input format: org.apache.hadoop.mapred.TextInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        ListSink
+

http://git-wip-us.apache.org/repos/asf/hive/blob/8d6f974a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardConstantStructObjectInspector.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardConstantStructObjectInspector.java
b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardConstantStructObjectInspector.java
new file mode 100644
index 0000000..8c47c0f
--- /dev/null
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardConstantStructObjectInspector.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2.objectinspector;
+
+import java.util.List;
+
+/**
+ * A StandardStructObjectInspector which also implements the
+ * ConstantObjectInspector interface.
+ *
+ * Always use the ObjectInspectorFactory to create new ObjectInspector objects,
+ * instead of directly creating an instance of this class.
+ */
+public class StandardConstantStructObjectInspector extends StandardStructObjectInspector
+  implements ConstantObjectInspector {
+
+  private List<?> value;
+
+  protected StandardConstantStructObjectInspector() {
+    super();
+  }
+  /**
+   * Call ObjectInspectorFactory.getStandardListObjectInspector instead.
+   */
+  protected StandardConstantStructObjectInspector(List<String> structFieldNames,
+	      List<ObjectInspector> structFieldObjectInspectors,  List<?> value) {
+    super(structFieldNames, structFieldObjectInspectors);
+    this.value = value;
+  }
+
+  @Override
+  public List<?> getWritableConstantValue() {
+    return value;
+  }
+}


Mime
View raw message