hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From gunt...@apache.org
Subject svn commit: r1552060 - in /hive/branches/tez: common/src/java/org/apache/hadoop/hive/conf/ ql/src/java/org/apache/hadoop/hive/ql/exec/ ql/src/java/org/apache/hadoop/hive/ql/optimizer/ ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/ ql/src/java/org...
Date Wed, 18 Dec 2013 18:56:10 GMT
Author: gunther
Date: Wed Dec 18 18:56:10 2013
New Revision: 1552060

URL: http://svn.apache.org/r1552060
Log:
HIVE-6055: Cleanup aisle tez (Gunther Hagleitner)

Modified:
    hive/branches/tez/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
    hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java
    hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/DemuxOperator.java
    hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java
    hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
    hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/PrunerOperatorFactory.java
    hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java
    hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
    hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java

Modified: hive/branches/tez/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1552060&r1=1552059&r2=1552060&view=diff
==============================================================================
--- hive/branches/tez/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/branches/tez/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Wed Dec 18
18:56:10 2013
@@ -872,7 +872,7 @@ public class HiveConf extends Configurat
     HIVE_RPC_QUERY_PLAN("hive.rpc.query.plan", false),
 
     // Whether to generate the splits locally or in the AM (tez only)
-      HIVE_AM_SPLIT_GENERATION("hive.compute.splits.in.am", true),
+    HIVE_AM_SPLIT_GENERATION("hive.compute.splits.in.am", true),
 
     // none, idonly, traverse, execution
     HIVESTAGEIDREARRANGE("hive.stageid.rearrange", "none"),

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java?rev=1552060&r1=1552059&r2=1552060&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java Wed
Dec 18 18:56:10 2013
@@ -34,8 +34,6 @@ import org.apache.hadoop.hive.ql.exec.pe
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.ql.plan.JoinCondDesc;
 import org.apache.hadoop.hive.ql.plan.JoinDesc;
-import org.apache.hadoop.hive.ql.plan.OperatorDesc;
-import org.apache.hadoop.hive.ql.plan.Statistics;
 import org.apache.hadoop.hive.ql.plan.TableDesc;
 import org.apache.hadoop.hive.serde2.io.ShortWritable;
 import org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe;

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/DemuxOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/DemuxOperator.java?rev=1552060&r1=1552059&r2=1552060&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/DemuxOperator.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/DemuxOperator.java Wed Dec
18 18:56:10 2013
@@ -27,11 +27,9 @@ import java.util.Map.Entry;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.ql.plan.DemuxDesc;
 import org.apache.hadoop.hive.ql.plan.OperatorDesc;
-import org.apache.hadoop.hive.ql.plan.Statistics;
 import org.apache.hadoop.hive.ql.plan.TableDesc;
 import org.apache.hadoop.hive.ql.plan.api.OperatorType;
 import org.apache.hadoop.hive.serde2.Deserializer;

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java?rev=1552060&r1=1552059&r2=1552060&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java Wed
Dec 18 18:56:10 2013
@@ -27,12 +27,10 @@ import java.util.Map;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.common.FileUtils;
-import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.common.StatsSetupConst;
 import org.apache.hadoop.hive.ql.ErrorMsg;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.ql.metadata.VirtualColumn;
-import org.apache.hadoop.hive.ql.plan.Statistics;
 import org.apache.hadoop.hive.ql.plan.OperatorDesc;
 import org.apache.hadoop.hive.ql.plan.TableDesc;
 import org.apache.hadoop.hive.ql.plan.TableScanDesc;

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java?rev=1552060&r1=1552059&r2=1552060&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java Wed Dec 18
18:56:10 2013
@@ -3163,59 +3163,6 @@ public final class Utilities {
     }
   }
 
-  public static long getSize(String alias, Table table, HiveConf conf,
-      TableScanOperator topOp, ExprNodeDesc expr) throws HiveException {
-    long result = 0;
-    int numPartitions = 0;
-    Map<String, PrunedPartitionList> prunedPartitionsMap
-      = new HashMap<String, PrunedPartitionList>();
-
-    if (!table.isPartitioned()) {
-      result = getSize(conf, table);
-    }
-    else {
-      // For partitioned tables, get the size of all the partitions
-      PrunedPartitionList partsList = PartitionPruner.prune(table, expr, conf,
-          alias, prunedPartitionsMap);
-      numPartitions = partsList.getNotDeniedPartns().size();
-      for (Partition part : partsList.getNotDeniedPartns()) {
-        result += getSize(conf, part);
-      }
-    }
-    return result;
-  }
-
-  private static long getSize(HiveConf conf, String size, Path path) {
-    // If the size is present in the metastore, use it
-    if (size != null) {
-      try {
-        return Long.valueOf(size);
-      } catch (NumberFormatException e) {
-        return -1;
-      }
-    }
-
-    try {
-      FileSystem fs = path.getFileSystem(conf);
-      return fs.getContentSummary(path).getLength();
-    } catch (Exception e) {
-      return -1;
-    }
-  }
-
-  private static long getSize(HiveConf conf, Table table) {
-    Path path = table.getPath();
-    String size = table.getProperty("totalSize");
-    return getSize(conf, size, path);
-  }
-
-  private static long getSize(HiveConf conf, Partition partition) {
-    Path path = partition.getPartitionPath();
-    String size = partition.getParameters().get("totalSize");
-
-    return getSize(conf, size, path);
-  }
-
   public static void clearWorkMap() {
     gWorkMap.clear();
   }

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/PrunerOperatorFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/PrunerOperatorFactory.java?rev=1552060&r1=1552059&r2=1552060&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/PrunerOperatorFactory.java
(original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/PrunerOperatorFactory.java
Wed Dec 18 18:56:10 2013
@@ -125,9 +125,6 @@ public abstract class PrunerOperatorFact
       // Put the mapping from table scan operator to pruner_pred
       opToPrunner.put(top, pruner_pred);
 
-      // Set the predicate in the table directly
-      top.getConf().setPruningPredicate(pruner_pred);
-
       return;
     }
 
@@ -168,9 +165,6 @@ public abstract class PrunerOperatorFact
       // Put the mapping from table scan operator to part-pruner map
       opToPrunner.put(top, partToPruner);
 
-      // Set the predicate in the table directly
-      top.getConf().setPruningPredicate(pruner_pred);
-
       return;
     }
   }

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java?rev=1552060&r1=1552059&r2=1552060&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java
(original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java
Wed Dec 18 18:56:10 2013
@@ -155,7 +155,7 @@ public class PartitionPruner implements 
    *         pruner condition.
    * @throws HiveException
    */
-  public static PrunedPartitionList prune(Table tab, ExprNodeDesc prunerExpr,
+  private static PrunedPartitionList prune(Table tab, ExprNodeDesc prunerExpr,
       HiveConf conf, String alias, Map<String, PrunedPartitionList> prunedPartitionsMap)
           throws HiveException {
     LOG.trace("Started pruning partiton");

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1552060&r1=1552059&r2=1552060&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Wed
Dec 18 18:56:10 2013
@@ -8352,9 +8352,6 @@ public class SemanticAnalyzer extends Ba
       // Add a mapping from the table scan operator to Table
       topToTable.put((TableScanOperator) top, tab);
 
-      // set the table in the tablescan descriptor directly
-      ((TableScanOperator) top).getConf().setTable(tab);
-
       Map<String, String> props = qb.getTabPropsForAlias(alias);
       if (props != null) {
         topToTableProps.put((TableScanOperator) top, props);

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java?rev=1552060&r1=1552059&r2=1552060&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java Wed Dec
18 18:56:10 2013
@@ -22,8 +22,6 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 
-import org.apache.hadoop.hive.ql.exec.PTFUtils;
-import org.apache.hadoop.hive.ql.metadata.Table;
 import org.apache.hadoop.hive.ql.metadata.VirtualColumn;
 
 /**
@@ -54,10 +52,6 @@ public class TableScanDesc extends Abstr
    */
   private int rowLimit = -1;
 
-  private transient Table table;
-
-  private transient ExprNodeDesc pruningPredicate;
-
   /**
    * A boolean variable set to true by the semantic analyzer only in case of the analyze
command.
    *
@@ -77,32 +71,10 @@ public class TableScanDesc extends Abstr
   // input file name (big) to bucket number
   private Map<String, Integer> bucketFileNameMapping;
 
-  static{
-    PTFUtils.makeTransient(TableScanDesc.class, "pruningPredicate");
-    PTFUtils.makeTransient(TableScanDesc.class, "table");
-  }
-
-
   @SuppressWarnings("nls")
   public TableScanDesc() {
   }
 
-  public Table getTable() {
-    return table;
-  }
-
-  public void setTable(Table t) {
-    table = t;
-  }
-
-  public ExprNodeDesc getPruningPredicate() {
-    return pruningPredicate;
-  }
-
-  public void setPruningPredicate(ExprNodeDesc expr) {
-    pruningPredicate = expr;
-  }
-
   public TableScanDesc(final String alias) {
     this.alias = alias;
   }



Mime
View raw message