hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ser...@apache.org
Subject hive git commit: HIVE-12057 : ORC sarg is logged too much (Sergey Shelukhin, reviewed by Ashutosh Chauhan)
Date Fri, 16 Oct 2015 20:38:30 GMT
Repository: hive
Updated Branches:
  refs/heads/master 20ad5b391 -> ea60008ff


HIVE-12057 : ORC sarg is logged too much (Sergey Shelukhin, reviewed by Ashutosh Chauhan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ea60008f
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ea60008f
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ea60008f

Branch: refs/heads/master
Commit: ea60008ff524af4a69fbf053027afa5d07a8e25e
Parents: 20ad5b3
Author: Sergey Shelukhin <sershe@apache.org>
Authored: Fri Oct 16 13:26:51 2015 -0700
Committer: Sergey Shelukhin <sershe@apache.org>
Committed: Fri Oct 16 13:26:51 2015 -0700

----------------------------------------------------------------------
 .../hadoop/hive/ql/io/orc/OrcInputFormat.java   | 16 ++--
 .../hive/ql/io/sarg/TestSearchArgumentImpl.java | 87 +++++++-------------
 .../hive/ql/io/sarg/SearchArgumentImpl.java     |  2 +-
 3 files changed, 41 insertions(+), 64 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/ea60008f/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
index b03e055..7c927dc 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
@@ -366,7 +366,9 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
       return;
     }
 
-    LOG.info("ORC pushdown predicate: " + sarg);
+    if (LOG.isInfoEnabled()) {
+      LOG.info("ORC pushdown predicate: " + sarg);
+    }
     options.searchArgument(sarg, getSargColumnNames(
         neededColumnNames.split(","), types, options.getInclude(), isOriginal));
   }
@@ -466,6 +468,7 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
     private final AtomicInteger numFilesCounter = new AtomicInteger(0);
     private ValidTxnList transactionList;
     private SplitStrategyKind splitStrategyKind;
+    private final SearchArgument sarg;
 
     Context(Configuration conf) {
       this(conf, 1);
@@ -473,6 +476,7 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
 
     Context(Configuration conf, final int minSplits) {
       this.conf = conf;
+      this.sarg = ConvertAstToSearchArg.createFromConf(conf);
       minSize = conf.getLong(MIN_SPLIT_SIZE, DEFAULT_MIN_SPLIT_SIZE);
       maxSize = conf.getLong(MAX_SPLIT_SIZE, DEFAULT_MAX_SPLIT_SIZE);
       String ss = conf.get(ConfVars.HIVE_ORC_SPLIT_STRATEGY.varname);
@@ -981,10 +985,10 @@ public class OrcInputFormat implements InputFormat<NullWritable,
OrcStruct>,
 
       // we can't eliminate stripes if there are deltas because the
       // deltas may change the rows making them match the predicate.
-      if (deltas.isEmpty() && canCreateSargFromConf(context.conf)) {
+      if ((deltas == null || deltas.isEmpty()) && context.sarg != null) {
         SearchArgument sarg = ConvertAstToSearchArg.createFromConf(context.conf);
-        String[] sargColNames = extractNeededColNames(types, context.conf, includedCols,
isOriginal);
-        includeStripe = pickStripes(sarg, sargColNames, writerVersion, isOriginal,
+        String[] colNames = extractNeededColNames(types, context.conf, includedCols, isOriginal);
+        includeStripe = pickStripes(context.sarg, colNames, writerVersion, isOriginal,
             stripeStats, stripes.size(), file.getPath());
       }
 
@@ -1110,6 +1114,9 @@ public class OrcInputFormat implements InputFormat<NullWritable,
OrcStruct>,
       conf = new HiveConf(conf, OrcInputFormat.class);
     }
     Context context = new Context(conf, numSplits);
+    if (LOG.isInfoEnabled()) {
+      LOG.info("ORC pushdown predicate: " + context.sarg);
+    }
     boolean useFileIds = HiveConf.getBoolVar(conf, ConfVars.HIVE_ORC_INCLUDE_FILE_ID_IN_SPLITS);
     List<OrcSplit> splits = Lists.newArrayList();
     List<Future<AcidDirInfo>> pathFutures = Lists.newArrayList();
@@ -1469,7 +1476,6 @@ public class OrcInputFormat implements InputFormat<NullWritable,
OrcStruct>,
   private static boolean[] pickStripes(SearchArgument sarg, String[] sargColNames,
       WriterVersion writerVersion, boolean isOriginal, List<StripeStatistics> stripeStats,
       int stripeCount, Path filePath) {
-    LOG.info("ORC pushdown predicate: " + sarg);
     if (sarg == null || stripeStats == null || writerVersion == OrcFile.WriterVersion.ORIGINAL)
{
       return null; // only do split pruning if HIVE-8732 has been fixed in the writer
     }

http://git-wip-us.apache.org/repos/asf/hive/blob/ea60008f/ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java b/ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java
index 573d5c6..8cbc26d 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java
@@ -28,21 +28,15 @@ import org.apache.hadoop.hive.common.type.HiveVarchar;
 import org.apache.hadoop.hive.ql.io.orc.TestInputOutputFormat;
 import org.apache.hadoop.hive.ql.io.sarg.SearchArgument.TruthValue;
 import org.apache.hadoop.hive.ql.io.sarg.SearchArgumentImpl.PredicateLeafImpl;
-import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
 import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
 import org.junit.Test;
 
-import java.beans.XMLDecoder;
-import java.io.ByteArrayInputStream;
-import java.io.UnsupportedEncodingException;
 import java.lang.reflect.Field;
 import java.sql.Date;
 import java.sql.Timestamp;
 import java.util.List;
 import java.util.Set;
 
-import org.apache.parquet.filter2.predicate.FilterPredicate;
-
 /**
  * These test the SARG implementation.
  * The xml files were generated by setting hive.optimize.index.filter
@@ -53,6 +47,7 @@ import org.apache.parquet.filter2.predicate.FilterPredicate;
  * In each case, the corresponding part of the where clause is in the
  * comment above the blob.
  */
+@SuppressWarnings("deprecation")
 public class TestSearchArgumentImpl {
 
   private ExpressionTree not(ExpressionTree arg) {
@@ -303,28 +298,6 @@ public class TestSearchArgumentImpl {
     }
   }
 
-  private ExprNodeGenericFuncDesc getFuncDesc(String xmlSerialized) {
-    byte[] bytes;
-    try {
-      bytes = xmlSerialized.getBytes("UTF-8");
-    } catch (UnsupportedEncodingException ex) {
-      throw new RuntimeException("UTF-8 support required", ex);
-    }
-
-    ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
-    XMLDecoder decoder = new XMLDecoder(bais, null, null);
-
-    try {
-      return (ExprNodeGenericFuncDesc) decoder.readObject();
-    } finally {
-      decoder.close();
-    }
-  }
-
-  private static TruthValue[] values(TruthValue... vals) {
-    return vals;
-  }
-
   @Test
   public void testBuilder() throws Exception {
     SearchArgument sarg =
@@ -335,9 +308,9 @@ public class TestSearchArgumentImpl {
             .equals("z", PredicateLeaf.Type.FLOAT, 1.0)
             .end()
             .build();
-    assertEquals("leaf-0 = (LESS_THAN x 10)\n" +
-        "leaf-1 = (LESS_THAN_EQUALS y hi)\n" +
-        "leaf-2 = (EQUALS z 1.0)\n" +
+    assertEquals("leaf-0 = (LESS_THAN x 10), " +
+        "leaf-1 = (LESS_THAN_EQUALS y hi), " +
+        "leaf-2 = (EQUALS z 1.0), " +
         "expr = (and leaf-0 leaf-1 leaf-2)", sarg.toString());
     sarg = SearchArgumentFactory.newBuilder()
         .startNot()
@@ -349,10 +322,10 @@ public class TestSearchArgumentImpl {
         .end()
         .end()
         .build();
-    assertEquals("leaf-0 = (IS_NULL x)\n" +
-        "leaf-1 = (BETWEEN y 10 20)\n" +
-        "leaf-2 = (IN z 1 2 3)\n" +
-        "leaf-3 = (NULL_SAFE_EQUALS a stinger)\n" +
+    assertEquals("leaf-0 = (IS_NULL x), " +
+        "leaf-1 = (BETWEEN y 10 20), " +
+        "leaf-2 = (IN z 1 2 3), " +
+        "leaf-3 = (NULL_SAFE_EQUALS a stinger), " +
         "expr = (and (not leaf-0) (not leaf-1) (not leaf-2) (not leaf-3))", sarg.toString());
   }
 
@@ -368,9 +341,9 @@ public class TestSearchArgumentImpl {
             .equals("z", PredicateLeaf.Type.DECIMAL, new HiveDecimalWritable("1.0"))
             .end()
             .build();
-    assertEquals("leaf-0 = (LESS_THAN x 1970-01-11)\n" +
-        "leaf-1 = (LESS_THAN_EQUALS y hi        )\n" +
-        "leaf-2 = (EQUALS z 1)\n" +
+    assertEquals("leaf-0 = (LESS_THAN x 1970-01-11), " +
+        "leaf-1 = (LESS_THAN_EQUALS y hi        ), " +
+        "leaf-2 = (EQUALS z 1), " +
         "expr = (and leaf-0 leaf-1 leaf-2)", sarg.toString());
 
     sarg = SearchArgumentFactory.newBuilder()
@@ -385,10 +358,10 @@ public class TestSearchArgumentImpl {
         .end()
         .end()
         .build();
-    assertEquals("leaf-0 = (IS_NULL x)\n" +
-        "leaf-1 = (BETWEEN y 10 20)\n" +
-        "leaf-2 = (IN z 1 2 3)\n" +
-        "leaf-3 = (NULL_SAFE_EQUALS a stinger)\n" +
+    assertEquals("leaf-0 = (IS_NULL x), " +
+        "leaf-1 = (BETWEEN y 10 20), " +
+        "leaf-2 = (IN z 1 2 3), " +
+        "leaf-3 = (NULL_SAFE_EQUALS a stinger), " +
         "expr = (and (not leaf-0) (not leaf-1) (not leaf-2) (not leaf-3))",
         sarg.toString());
   }
@@ -405,9 +378,9 @@ public class TestSearchArgumentImpl {
                 new HiveDecimalWritable("1.0"))
             .end()
             .build();
-    assertEquals("leaf-0 = (LESS_THAN x 2005-03-12)\n" +
-        "leaf-1 = (LESS_THAN_EQUALS y hi        )\n" +
-        "leaf-2 = (EQUALS z 1)\n" +
+    assertEquals("leaf-0 = (LESS_THAN x 2005-03-12), " +
+        "leaf-1 = (LESS_THAN_EQUALS y hi        ), " +
+        "leaf-2 = (EQUALS z 1), " +
         "expr = (and leaf-0 leaf-1 leaf-2)", sarg.toString());
 
     sarg = SearchArgumentFactory.newBuilder()
@@ -422,10 +395,10 @@ public class TestSearchArgumentImpl {
         .end()
         .end()
         .build();
-    assertEquals("leaf-0 = (IS_NULL x)\n" +
-        "leaf-1 = (BETWEEN y 10 20)\n" +
-        "leaf-2 = (IN z 1 2 3)\n" +
-        "leaf-3 = (NULL_SAFE_EQUALS a stinger)\n" +
+    assertEquals("leaf-0 = (IS_NULL x), " +
+        "leaf-1 = (BETWEEN y 10 20), " +
+        "leaf-2 = (IN z 1 2 3), " +
+        "leaf-3 = (NULL_SAFE_EQUALS a stinger), " +
         "expr = (and (not leaf-0) (not leaf-1) (not leaf-2) (not leaf-3))",
         sarg.toString());
   }
@@ -443,11 +416,11 @@ public class TestSearchArgumentImpl {
             .equals("z1", PredicateLeaf.Type.FLOAT, new Double(0.22))
             .end()
             .build();
-    assertEquals("leaf-0 = (LESS_THAN x 22)\n" +
-        "leaf-1 = (LESS_THAN x1 22)\n" +
-        "leaf-2 = (LESS_THAN_EQUALS y hi        )\n" +
-        "leaf-3 = (EQUALS z 0.22)\n" +
-        "leaf-4 = (EQUALS z1 0.22)\n" +
+    assertEquals("leaf-0 = (LESS_THAN x 22), " +
+        "leaf-1 = (LESS_THAN x1 22), " +
+        "leaf-2 = (LESS_THAN_EQUALS y hi        ), " +
+        "leaf-3 = (EQUALS z 0.22), " +
+        "leaf-4 = (EQUALS z1 0.22), " +
         "expr = (and leaf-0 leaf-1 leaf-2 leaf-3 leaf-4)", sarg.toString());
   }
 
@@ -477,8 +450,7 @@ public class TestSearchArgumentImpl {
 
   @Test(expected = IllegalArgumentException.class)
   public void testBadLiteral() throws Exception {
-    SearchArgument sarg =
-        SearchArgumentFactory.newBuilder()
+    SearchArgumentFactory.newBuilder()
         .startAnd()
         .lessThan("x", PredicateLeaf.Type.LONG, "hi")
         .end()
@@ -487,8 +459,7 @@ public class TestSearchArgumentImpl {
 
   @Test(expected = IllegalArgumentException.class)
   public void testBadLiteralList() throws Exception {
-    SearchArgument sarg =
-        SearchArgumentFactory.newBuilder()
+    SearchArgumentFactory.newBuilder()
             .startAnd()
             .in("x", PredicateLeaf.Type.STRING, "hi", 23)
             .end()

http://git-wip-us.apache.org/repos/asf/hive/blob/ea60008f/storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java
b/storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java
index a762b8b..4c3dd5a 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java
@@ -211,7 +211,7 @@ final class SearchArgumentImpl implements SearchArgument {
       buffer.append(i);
       buffer.append(" = ");
       buffer.append(leaves.get(i).toString());
-      buffer.append('\n');
+      buffer.append(", ");
     }
     buffer.append("expr = ");
     buffer.append(expression);


Mime
View raw message