hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From br...@apache.org
Subject svn commit: r1536501 [1/2] - in /hive/branches/maven: ./ ql/src/java/org/apache/hadoop/hive/ql/exec/ ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ ql/src/java/org/apache/hadoop/hive/ql/io/ ql/src/java/org/apache/hadoop/hive/ql/udf/ ql/src/test/org/apa...
Date Mon, 28 Oct 2013 20:12:50 GMT
Author: brock
Date: Mon Oct 28 20:12:49 2013
New Revision: 1536501

URL: http://svn.apache.org/r1536501
Log:
Merge trunk into branch

Added:
    hive/branches/maven/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFOPMod.java
      - copied unchanged from r1536499, hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFOPMod.java
    hive/branches/maven/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFPosMod.java
      - copied unchanged from r1536499, hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFPosMod.java
Modified:
    hive/branches/maven/   (props changed)
    hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
    hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java
    hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java
    hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java
    hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecMapperContext.java
    hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/HiveContextAwareRecordReader.java
    hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/IOContext.java
    hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPMod.java
    hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFPosMod.java
    hive/branches/maven/ql/src/test/results/clientpositive/vectorization_14.q.out

Propchange: hive/branches/maven/
------------------------------------------------------------------------------
  Merged /hive/trunk:r1536475-1536499

Modified: hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java?rev=1536501&r1=1536500&r2=1536501&view=diff
==============================================================================
--- hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java (original)
+++ hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java Mon
Oct 28 20:12:49 2013
@@ -120,7 +120,7 @@ public class MapJoinOperator extends Abs
     }
 
     String baseDir = null;
-    String currentInputFile = getExecContext().getCurrentInputFile();
+    String currentInputFile = getExecContext().getCurrentInputPath().toString();
     LOG.info("******* Load from HashTable File: input : " + currentInputFile);
     String fileName = getExecContext().getLocalWork().getBucketFileName(currentInputFile);
     try {

Modified: hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java?rev=1536501&r1=1536500&r2=1536501&view=diff
==============================================================================
--- hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java (original)
+++ hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java Mon Oct
28 20:12:49 2013
@@ -441,7 +441,7 @@ public class MapOperator extends Operato
   // Find context for current input file
   @Override
   public void cleanUpInputFileChangedOp() throws HiveException {
-    Path fpath = normalizePath(getExecContext().getCurrentInputFile());
+    Path fpath = getExecContext().getCurrentInputPath();
 
     for (String onefile : conf.getPathToAliases().keySet()) {
       Path onepath = normalizePath(onefile);
@@ -537,7 +537,7 @@ public class MapOperator extends Operato
       VirtualColumn vc = vcs.get(i);
       if (vc.equals(VirtualColumn.FILENAME)) {
         if (ctx.inputFileChanged()) {
-          vcValues[i] = new Text(ctx.getCurrentInputFile());
+          vcValues[i] = new Text(ctx.getCurrentInputPath().toString());
         }
       } else if (vc.equals(VirtualColumn.BLOCKOFFSET)) {
         long current = ctx.getIoCxt().getCurrentBlockStart();

Modified: hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java?rev=1536501&r1=1536500&r2=1536501&view=diff
==============================================================================
--- hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java
(original)
+++ hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java
Mon Oct 28 20:12:49 2013
@@ -515,19 +515,19 @@ public class SMBMapJoinOperator extends 
   private void setUpFetchContexts(String alias, MergeQueue mergeQueue) throws HiveException
{
     mergeQueue.clearFetchContext();
 
-    String currentInputFile = getExecContext().getCurrentInputFile();
+    Path currentInputPath = getExecContext().getCurrentInputPath();
 
     BucketMapJoinContext bucketMatcherCxt = localWork.getBucketMapjoinContext();
     Class<? extends BucketMatcher> bucketMatcherCls = bucketMatcherCxt.getBucketMatcherClass();
     BucketMatcher bucketMatcher = ReflectionUtils.newInstance(bucketMatcherCls, null);
 
-    getExecContext().setFileId(bucketMatcherCxt.createFileId(currentInputFile));
+    getExecContext().setFileId(bucketMatcherCxt.createFileId(currentInputPath.toString()));
     LOG.info("set task id: " + getExecContext().getFileId());
 
     bucketMatcher.setAliasBucketFileNameMapping(bucketMatcherCxt
         .getAliasBucketFileNameMapping());
 
-    List<Path> aliasFiles = bucketMatcher.getAliasBucketFiles(currentInputFile,
+    List<Path> aliasFiles = bucketMatcher.getAliasBucketFiles(currentInputPath.toString(),
         bucketMatcherCxt.getMapJoinBigTableAlias(), alias);
 
     mergeQueue.setupContext(aliasFiles);

Modified: hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java?rev=1536501&r1=1536500&r2=1536501&view=diff
==============================================================================
--- hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java
(original)
+++ hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java
Mon Oct 28 20:12:49 2013
@@ -105,9 +105,9 @@ public class TableScanOperator extends O
     Map<String, Integer> bucketNameMapping =
         (conf != null) ? conf.getBucketFileNameMapping() : null;
     if ((bucketNameMapping != null) && (!bucketNameMapping.isEmpty())) {
-      String currentInputFile = getExecContext().getCurrentInputFile();
+      Path currentInputPath = getExecContext().getCurrentInputPath();
       getExecContext().setFileId(Integer.toString(bucketNameMapping.get(
-          Utilities.getFileNameFromDirName(currentInputFile))));
+          currentInputPath.getName())));
     }
   }
 

Modified: hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecMapperContext.java
URL: http://svn.apache.org/viewvc/hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecMapperContext.java?rev=1536501&r1=1536500&r2=1536501&view=diff
==============================================================================
--- hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecMapperContext.java
(original)
+++ hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecMapperContext.java
Mon Oct 28 20:12:49 2013
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.exec.m
 import java.util.Map;
 
 import org.apache.commons.logging.Log;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.ql.exec.FetchOperator;
 import org.apache.hadoop.hive.ql.io.IOContext;
 import org.apache.hadoop.hive.ql.plan.MapredLocalWork;
@@ -29,14 +30,14 @@ public class ExecMapperContext {
 
   public static final Log l4j = ExecMapper.l4j;
 
-  // lastInputFile should be changed by the root of the operator tree ExecMapper.map()
+  // lastInputPath should be changed by the root of the operator tree ExecMapper.map()
   // but kept unchanged throughout the operator tree for one row
-  private String lastInputFile = null;
+  private Path lastInputPath = null;
 
   // currentInputFile will be updated only by inputFileChanged(). If inputFileChanged()
-  // is not called throughout the opertor tree, currentInputFile won't be used anyways
+  // is not called throughout the operator tree, currentInputPath won't be used anyways
   // so it won't be updated.
-  private String currentInputFile = null;
+  private Path currentInputPath = null;
   private boolean inputFileChecked = false;
 
   // for SMB join, replaced with number part of task-id , making output file name
@@ -80,10 +81,10 @@ public class ExecMapperContext {
    */
   public boolean inputFileChanged() {
     if (!inputFileChecked) {
-      currentInputFile = this.ioCxt.getInputFile();
+      currentInputPath = this.ioCxt.getInputPath();
       inputFileChecked = true;
     }
-    return lastInputFile == null || !lastInputFile.equals(currentInputFile);
+    return lastInputPath == null || !lastInputPath.equals(currentInputPath);
   }
 
   /**
@@ -93,25 +94,25 @@ public class ExecMapperContext {
    */
   public void resetRow() {
     // Update the lastInputFile with the currentInputFile.
-    lastInputFile = currentInputFile;
+    lastInputPath = currentInputPath;
     inputFileChecked = false;
   }
 
-  public String getLastInputFile() {
-    return lastInputFile;
+  public Path getLastInputPath() {
+    return lastInputPath;
   }
 
-  public void setLastInputFile(String lastInputFile) {
-    this.lastInputFile = lastInputFile;
+  public void setLastInputPath(Path lastInputPath) {
+    this.lastInputPath = lastInputPath;
   }
 
-  public String getCurrentInputFile() {
-    currentInputFile = this.ioCxt.getInputFile();
-    return currentInputFile;
+  public Path getCurrentInputPath() {
+    currentInputPath = this.ioCxt.getInputPath();
+    return currentInputPath;
   }
 
-  public void setCurrentInputFile(String currentInputFile) {
-    this.currentInputFile = currentInputFile;
+  public void setCurrentInputPath(Path currentInputPath) {
+    this.currentInputPath = currentInputPath;
   }
 
   public JobConf getJc() {

Modified: hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/HiveContextAwareRecordReader.java
URL: http://svn.apache.org/viewvc/hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/HiveContextAwareRecordReader.java?rev=1536501&r1=1536500&r2=1536501&view=diff
==============================================================================
--- hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/HiveContextAwareRecordReader.java
(original)
+++ hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/HiveContextAwareRecordReader.java
Mon Oct 28 20:12:49 2013
@@ -146,12 +146,12 @@ public abstract class HiveContextAwareRe
     return IOContext.get();
   }
 
-  public void initIOContext(long startPos, boolean isBlockPointer, String inputFile) {
+  public void initIOContext(long startPos, boolean isBlockPointer, Path inputPath) {
     ioCxtRef = this.getIOContext();
     ioCxtRef.currentBlockStart = startPos;
     ioCxtRef.isBlockPointer = isBlockPointer;
-    ioCxtRef.inputFile = inputFile;
-    LOG.info("Processing file " + inputFile);
+    ioCxtRef.inputPath = inputPath;
+    LOG.info("Processing file " + inputPath);
     initDone = true;
   }
 
@@ -184,7 +184,7 @@ public abstract class HiveContextAwareRe
       blockStart = in.getPosition();
       in.close();
     }
-    this.initIOContext(blockStart, blockPointer, path.makeQualified(fs).toString());
+    this.initIOContext(blockStart, blockPointer, path.makeQualified(fs));
 
     this.initIOContextSortedProps(split, recordReader, job);
   }

Modified: hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/IOContext.java
URL: http://svn.apache.org/viewvc/hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/IOContext.java?rev=1536501&r1=1536500&r2=1536501&view=diff
==============================================================================
--- hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/IOContext.java (original)
+++ hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/IOContext.java Mon Oct 28
20:12:49 2013
@@ -18,6 +18,8 @@
 
 package org.apache.hadoop.hive.ql.io;
 
+import org.apache.hadoop.fs.Path;
+
 
 /**
  * IOContext basically contains the position information of the current
@@ -67,7 +69,7 @@ public class IOContext {
     UNKNOWN
   }
 
-  String inputFile;
+  Path inputPath;
 
   public IOContext() {
     this.currentBlockStart = 0;
@@ -109,12 +111,12 @@ public class IOContext {
     this.isBlockPointer = isBlockPointer;
   }
 
-  public String getInputFile() {
-    return inputFile;
+  public Path getInputPath() {
+    return inputPath;
   }
 
-  public void setInputFile(String inputFile) {
-    this.inputFile = inputFile;
+  public void setInputPath(Path inputPath) {
+    this.inputPath = inputPath;
   }
 
   public void setIOExceptions(boolean ioe) {

Modified: hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPMod.java
URL: http://svn.apache.org/viewvc/hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPMod.java?rev=1536501&r1=1536500&r2=1536501&view=diff
==============================================================================
--- hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPMod.java (original)
+++ hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPMod.java Mon Oct 28
20:12:49 2013
@@ -61,7 +61,7 @@ public class UDFOPMod extends UDFBaseNum
   public ByteWritable evaluate(ByteWritable a, ByteWritable b) {
     // LOG.info("Get input " + a.getClass() + ":" + a + " " + b.getClass() + ":"
     // + b);
-    if ((a == null) || (b == null)) {
+    if (a == null || b == null || b.get() == 0) {
       return null;
     }
 
@@ -73,7 +73,7 @@ public class UDFOPMod extends UDFBaseNum
   public ShortWritable evaluate(ShortWritable a, ShortWritable b) {
     // LOG.info("Get input " + a.getClass() + ":" + a + " " + b.getClass() + ":"
     // + b);
-    if ((a == null) || (b == null)) {
+    if (a == null || b == null || b.get() == 0) {
       return null;
     }
 
@@ -85,7 +85,7 @@ public class UDFOPMod extends UDFBaseNum
   public IntWritable evaluate(IntWritable a, IntWritable b) {
     // LOG.info("Get input " + a.getClass() + ":" + a + " " + b.getClass() + ":"
     // + b);
-    if ((a == null) || (b == null)) {
+    if (a == null || b == null || b.get() == 0) {
       return null;
     }
 
@@ -97,7 +97,7 @@ public class UDFOPMod extends UDFBaseNum
   public LongWritable evaluate(LongWritable a, LongWritable b) {
     // LOG.info("Get input " + a.getClass() + ":" + a + " " + b.getClass() + ":"
     // + b);
-    if ((a == null) || (b == null)) {
+    if (a == null || b == null || b.get() == 0L) {
       return null;
     }
 
@@ -109,7 +109,7 @@ public class UDFOPMod extends UDFBaseNum
   public FloatWritable evaluate(FloatWritable a, FloatWritable b) {
     // LOG.info("Get input " + a.getClass() + ":" + a + " " + b.getClass() + ":"
     // + b);
-    if ((a == null) || (b == null)) {
+    if (a == null || b == null || b.get() == 0.0f) {
       return null;
     }
 
@@ -121,7 +121,7 @@ public class UDFOPMod extends UDFBaseNum
   public DoubleWritable evaluate(DoubleWritable a, DoubleWritable b) {
     // LOG.info("Get input " + a.getClass() + ":" + a + " " + b.getClass() + ":"
     // + b);
-    if ((a == null) || (b == null)) {
+    if (a == null || b == null || b.get() == 0.0) {
       return null;
     }
 

Modified: hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFPosMod.java
URL: http://svn.apache.org/viewvc/hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFPosMod.java?rev=1536501&r1=1536500&r2=1536501&view=diff
==============================================================================
--- hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFPosMod.java (original)
+++ hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFPosMod.java Mon Oct 28
20:12:49 2013
@@ -47,7 +47,7 @@ public class UDFPosMod extends UDFBaseNu
   public ByteWritable evaluate(ByteWritable a, ByteWritable b) {
     // LOG.info("Get input " + a.getClass() + ":" + a + " " + b.getClass() + ":"
     // + b);
-    if ((a == null) || (b == null)) {
+    if (a == null || b == null || b.get() == 0) {
       return null;
     }
 
@@ -59,7 +59,7 @@ public class UDFPosMod extends UDFBaseNu
   public ShortWritable evaluate(ShortWritable a, ShortWritable b) {
     // LOG.info("Get input " + a.getClass() + ":" + a + " " + b.getClass() + ":"
     // + b);
-    if ((a == null) || (b == null)) {
+    if (a == null || b == null || b.get() == 0) {
       return null;
     }
 
@@ -71,7 +71,7 @@ public class UDFPosMod extends UDFBaseNu
   public IntWritable evaluate(IntWritable a, IntWritable b) {
     // LOG.info("Get input " + a.getClass() + ":" + a + " " + b.getClass() + ":"
     // + b);
-    if ((a == null) || (b == null)) {
+    if (a == null || b == null || b.get() == 0) {
       return null;
     }
 
@@ -83,7 +83,7 @@ public class UDFPosMod extends UDFBaseNu
   public LongWritable evaluate(LongWritable a, LongWritable b) {
     // LOG.info("Get input " + a.getClass() + ":" + a + " " + b.getClass() + ":"
     // + b);
-    if ((a == null) || (b == null)) {
+    if (a == null || b == null || b.get() == 0L) {
       return null;
     }
 
@@ -95,7 +95,7 @@ public class UDFPosMod extends UDFBaseNu
   public FloatWritable evaluate(FloatWritable a, FloatWritable b) {
     // LOG.info("Get input " + a.getClass() + ":" + a + " " + b.getClass() + ":"
     // + b);
-    if ((a == null) || (b == null)) {
+    if (a == null || b == null || b.get() == 0.0f) {
       return null;
     }
 
@@ -107,7 +107,7 @@ public class UDFPosMod extends UDFBaseNu
   public DoubleWritable evaluate(DoubleWritable a, DoubleWritable b) {
     // LOG.info("Get input " + a.getClass() + ":" + a + " " + b.getClass() + ":"
     // + b);
-    if ((a == null) || (b == null)) {
+    if (a == null || b == null || b.get() == 0.0) {
       return null;
     }
 



Mime
View raw message