hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1462670 [2/14] - in /hive/branches/ptf-windowing: ./ common/src/java/org/apache/hadoop/hive/conf/ conf/ contrib/src/java/org/apache/hadoop/hive/contrib/serde2/ contrib/src/java/org/apache/hadoop/hive/contrib/serde2/s3/ data/conf/ data/file...
Date Fri, 29 Mar 2013 22:59:41 GMT

Propchange: hive/branches/ptf-windowing/
------------------------------------------------------------------------------
  Merged /hive/trunk:r1458525-1462327

Modified: hive/branches/ptf-windowing/build-common.xml
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/build-common.xml?rev=1462670&r1=1462669&r2=1462670&view=diff
==============================================================================
--- hive/branches/ptf-windowing/build-common.xml (original)
+++ hive/branches/ptf-windowing/build-common.xml Fri Mar 29 22:58:43 2013
@@ -463,7 +463,7 @@
       <sysproperty key="java.net.preferIPv4Stack" value="${java.net.preferIPv4Stack}"/>
       <sysproperty key="hadoop.bin.path" value="${test.hadoop.bin.path}${junit.script.extension}"/>
       <sysproperty key="test.concurrency.num.threads" value="${test.concurrency.num.threads}"/>
-      <jvmarg line="${jvm.args}"/>
+      <jvmarg line="${junit.jvm.args}"/>
 
       <classpath refid="test.local.classpath"/>
       <formatter type="${test.junit.output.format}" usefile="${test.junit.output.usefile}" />

Modified: hive/branches/ptf-windowing/build.properties
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/build.properties?rev=1462670&r1=1462669&r2=1462670&view=diff
==============================================================================
--- hive/branches/ptf-windowing/build.properties (original)
+++ hive/branches/ptf-windowing/build.properties Fri Mar 29 22:58:43 2013
@@ -141,6 +141,9 @@ datanucleus.repo=http://www.datanucleus.
 # JVM arguments
 jvm.args=-XX:-UseSplitVerifier
 
+# junit jvm args
+junit.jvm.args=-XX:-UseSplitVerifier -XX:+CMSClassUnloadingEnabled -XX:MaxPermSize=128M
+
 #
 # Eclipse Properties
 #

Modified: hive/branches/ptf-windowing/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1462670&r1=1462669&r2=1462670&view=diff
==============================================================================
--- hive/branches/ptf-windowing/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/branches/ptf-windowing/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Fri Mar 29 22:58:43 2013
@@ -223,6 +223,9 @@ public class HiveConf extends Configurat
     // not specified
     DROPIGNORESNONEXISTENT("hive.exec.drop.ignorenonexistent", true),
 
+    // ignore the mapjoin hint
+    HIVEIGNOREMAPJOINHINT("hive.ignore.mapjoin.hint", true),
+
     // Hadoop Configuration Properties
     // Properties with null values are ignored and exist only for the purpose of giving us
     // a symbolic name to reference in the Hive source code. Properties with non-null
@@ -484,7 +487,7 @@ public class HiveConf extends Configurat
 
     HIVESKEWJOIN("hive.optimize.skewjoin", false),
     HIVECONVERTJOIN("hive.auto.convert.join", true),
-    HIVECONVERTJOINNOCONDITIONALTASK("hive.auto.convert.join.noconditionaltask", false),
+    HIVECONVERTJOINNOCONDITIONALTASK("hive.auto.convert.join.noconditionaltask", true),
     HIVECONVERTJOINNOCONDITIONALTASKTHRESHOLD("hive.auto.convert.join.noconditionaltask.size",
         10000000L),
     HIVESKEWJOINKEY("hive.skewjoin.key", 100000),
@@ -603,6 +606,7 @@ public class HiveConf extends Configurat
     // higher compute cost.
     HIVE_STATS_NDV_ERROR("hive.stats.ndv.error", (float)20.0),
     HIVE_STATS_KEY_PREFIX_MAX_LENGTH("hive.stats.key.prefix.max.length", 200),
+    HIVE_STATS_KEY_PREFIX("hive.stats.key.prefix", ""), // internal usage only
 
     // Concurrency
     HIVE_SUPPORT_CONCURRENCY("hive.support.concurrency", false),

Modified: hive/branches/ptf-windowing/conf/hive-default.xml.template
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/conf/hive-default.xml.template?rev=1462670&r1=1462669&r2=1462670&view=diff
==============================================================================
--- hive/branches/ptf-windowing/conf/hive-default.xml.template (original)
+++ hive/branches/ptf-windowing/conf/hive-default.xml.template Fri Mar 29 22:58:43 2013
@@ -785,6 +785,12 @@
 </property>
 
 <property>
+  <name>hive.ignore.mapjoin.hint</name>
+  <value>true</value>
+  <description>Ignore the mapjoin hint</description>
+</property>
+
+<property>
   <name>hive.mapjoin.localtask.max.memory.usage</name>
   <value>0.90</value>
   <description>This number means how much memory the local task can take to hold the key/value into in-memory hash table; If the local task's memory usage is more than this number, the local task will be abort by themself. It means the data of small table is too large to be hold in the memory.</description>
@@ -810,7 +816,7 @@
 
 <property>
   <name>hive.auto.convert.join.noconditionaltask</name>
-  <value>false</value>
+  <value>true</value>
   <description>Whether Hive enable the optimization about converting common join into mapjoin based on the input file 
     size. If this paramater is on, and the sum of size for n-1 of the tables/partitions for a n-way join is smaller than the
     specified size, the join is directly converted to a mapjoin (there is no conditional task).

Modified: hive/branches/ptf-windowing/contrib/src/java/org/apache/hadoop/hive/contrib/serde2/RegexSerDe.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/contrib/src/java/org/apache/hadoop/hive/contrib/serde2/RegexSerDe.java?rev=1462670&r1=1462669&r2=1462670&view=diff
==============================================================================
--- hive/branches/ptf-windowing/contrib/src/java/org/apache/hadoop/hive/contrib/serde2/RegexSerDe.java (original)
+++ hive/branches/ptf-windowing/contrib/src/java/org/apache/hadoop/hive/contrib/serde2/RegexSerDe.java Fri Mar 29 22:58:43 2013
@@ -29,7 +29,7 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.serde.serdeConstants;
-import org.apache.hadoop.hive.serde2.SerDe;
+import org.apache.hadoop.hive.serde2.AbstractSerDe;
 import org.apache.hadoop.hive.serde2.SerDeException;
 import org.apache.hadoop.hive.serde2.SerDeStats;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -70,7 +70,7 @@ import org.apache.hadoop.io.Writable;
  * writableStringObjectInspector. We should switch to that when we have a UTF-8
  * based Regex library.
  */
-public class RegexSerDe implements SerDe {
+public class RegexSerDe extends AbstractSerDe {
 
   public static final Log LOG = LogFactory.getLog(RegexSerDe.class.getName());
 

Modified: hive/branches/ptf-windowing/contrib/src/java/org/apache/hadoop/hive/contrib/serde2/TypedBytesSerDe.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/contrib/src/java/org/apache/hadoop/hive/contrib/serde2/TypedBytesSerDe.java?rev=1462670&r1=1462669&r2=1462670&view=diff
==============================================================================
--- hive/branches/ptf-windowing/contrib/src/java/org/apache/hadoop/hive/contrib/serde2/TypedBytesSerDe.java (original)
+++ hive/branches/ptf-windowing/contrib/src/java/org/apache/hadoop/hive/contrib/serde2/TypedBytesSerDe.java Fri Mar 29 22:58:43 2013
@@ -32,7 +32,7 @@ import org.apache.hadoop.hive.contrib.ut
 import org.apache.hadoop.hive.ql.io.NonSyncDataInputBuffer;
 import org.apache.hadoop.hive.ql.io.NonSyncDataOutputBuffer;
 import org.apache.hadoop.hive.serde.serdeConstants;
-import org.apache.hadoop.hive.serde2.SerDe;
+import org.apache.hadoop.hive.serde2.AbstractSerDe;
 import org.apache.hadoop.hive.serde2.SerDeException;
 import org.apache.hadoop.hive.serde2.SerDeStats;
 import org.apache.hadoop.hive.serde2.SerDeUtils;
@@ -40,11 +40,11 @@ import org.apache.hadoop.hive.serde2.io.
 import org.apache.hadoop.hive.serde2.io.DoubleWritable;
 import org.apache.hadoop.hive.serde2.io.ShortWritable;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
 import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.StructField;
 import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector;
@@ -73,7 +73,7 @@ import org.apache.hadoop.io.Writable;
  * this, which is apparently 25% faster than the python version is available at
  * http://github.com/klbostee/ctypedbytes/tree/master
  */
-public class TypedBytesSerDe implements SerDe {
+public class TypedBytesSerDe extends AbstractSerDe {
 
   public static final Log LOG = LogFactory.getLog(TypedBytesSerDe.class
       .getName());

Modified: hive/branches/ptf-windowing/contrib/src/java/org/apache/hadoop/hive/contrib/serde2/s3/S3LogDeserializer.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/contrib/src/java/org/apache/hadoop/hive/contrib/serde2/s3/S3LogDeserializer.java?rev=1462670&r1=1462669&r2=1462670&view=diff
==============================================================================
--- hive/branches/ptf-windowing/contrib/src/java/org/apache/hadoop/hive/contrib/serde2/s3/S3LogDeserializer.java (original)
+++ hive/branches/ptf-windowing/contrib/src/java/org/apache/hadoop/hive/contrib/serde2/s3/S3LogDeserializer.java Fri Mar 29 22:58:43 2013
@@ -27,7 +27,7 @@ import java.util.regex.Pattern;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.serde2.Deserializer;
+import org.apache.hadoop.hive.serde2.AbstractDeserializer;
 import org.apache.hadoop.hive.serde2.SerDeException;
 import org.apache.hadoop.hive.serde2.SerDeStats;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -42,7 +42,7 @@ import org.apache.hadoop.io.Writable;
  * S3LogDeserializer.
  *
  */
-public class S3LogDeserializer implements Deserializer {
+public class S3LogDeserializer extends AbstractDeserializer {
 
   public static final Log LOG = LogFactory.getLog(S3LogDeserializer.class
       .getName());
@@ -73,6 +73,7 @@ public class S3LogDeserializer implement
 
   S3LogStruct deserializeCache = new S3LogStruct();
 
+  @Override
   public void initialize(Configuration job, Properties tbl)
       throws SerDeException {
 
@@ -132,6 +133,7 @@ public class S3LogDeserializer implement
     return (c);
   }
 
+  @Override
   public Object deserialize(Writable field) throws SerDeException {
     String row = null;
     if (field instanceof BytesWritable) {
@@ -155,6 +157,7 @@ public class S3LogDeserializer implement
     }
   }
 
+  @Override
   public ObjectInspector getObjectInspector() throws SerDeException {
     return cachedObjectInspector;
   }
@@ -202,6 +205,7 @@ public class S3LogDeserializer implement
 
   }
 
+  @Override
   public SerDeStats getSerDeStats() {
     // no support for statistics
     return null;

Modified: hive/branches/ptf-windowing/data/conf/hive-site.xml
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/data/conf/hive-site.xml?rev=1462670&r1=1462669&r2=1462670&view=diff
==============================================================================
--- hive/branches/ptf-windowing/data/conf/hive-site.xml (original)
+++ hive/branches/ptf-windowing/data/conf/hive-site.xml Fri Mar 29 22:58:43 2013
@@ -177,6 +177,12 @@
 </property>
 
 <property>
+  <name>hive.ignore.mapjoin.hint</name>
+  <value>false</value>
+  <description>Whether Hive ignores the mapjoin hint</description>
+</property>
+
+<property>
   <name>hive.input.format</name>
   <value>org.apache.hadoop.hive.ql.io.CombineHiveInputFormat</value>
   <description>The default input format, if it is not specified, the system assigns it. It is set to HiveInputFormat for hadoop versions 17, 18 and 19, whereas it is set to CombineHiveInputFormat for hadoop 20. The user can always overwrite it - if there is a bug in CombineHiveInputFormat, it can always be manually set to HiveInputFormat. </description>

Modified: hive/branches/ptf-windowing/eclipse-templates/.classpath
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/eclipse-templates/.classpath?rev=1462670&r1=1462669&r2=1462670&view=diff
==============================================================================
--- hive/branches/ptf-windowing/eclipse-templates/.classpath (original)
+++ hive/branches/ptf-windowing/eclipse-templates/.classpath Fri Mar 29 22:58:43 2013
@@ -45,6 +45,7 @@
   <classpathentry kind="lib" path="build/ivy/lib/hadoop0.20.shim/oro-2.0.8.jar"/>
   <classpathentry kind="lib" path="build/ivy/lib/hadoop0.20.shim/servlet-api-2.5-6.1.14.jar"/>
   <classpathentry kind="lib" path="build/ivy/lib/hadoop0.20.shim/xmlenc-0.52.jar"/>
+  <classpathentry kind="lib" path="build/ivy/lib/hadoop0.20.shim/guava-@guava-hadoop20.version@.jar"/>
   <classpathentry kind="lib" path="build/ivy/lib/default/avro-@avro.version@.jar"/>
   <classpathentry kind="lib" path="build/ivy/lib/default/avro-mapred-@avro.version@.jar"/>
   <classpathentry kind="lib" path="build/ivy/lib/default/jline-@jline.version@.jar"/>
@@ -57,7 +58,6 @@
   <classpathentry kind="lib" path="build/ivy/lib/default/commons-logging-api-@commons-logging-api.version@.jar"/>
   <classpathentry kind="lib" path="build/ivy/lib/default/commons-io-@commons-io.version@.jar"/>
   <classpathentry kind="lib" path="build/ivy/lib/default/derby-@derby.version@.jar"/>
-  <classpathentry kind="lib" path="build/ivy/lib/default/guava-@guava-hadoop20.version@.jar"/>
   <classpathentry kind="lib" path="build/ivy/lib/default/hbase-@hbase.version@.jar"/>
   <classpathentry kind="lib" path="build/ivy/lib/default/hbase-@hbase.version@-tests.jar"/>
   <classpathentry kind="lib" path="build/ivy/lib/default/libfb303-@libfb303.version@.jar"/>
@@ -86,12 +86,14 @@
   <classpathentry kind="lib" path="build/ivy/lib/default/jackson-mapper-asl-1.8.8.jar"/>
   <classpathentry kind="lib" path="build/ivy/lib/default/jackson-xc-1.8.8.jar"/>
   <classpathentry kind="lib" path="build/ivy/lib/default/mockito-all-@mockito-all.version@.jar"/>
-  <classpathentry kind="lib" path="build/ivy/lib/default/stringtemplate-@stringtemplate.version@.jar"/>
+  <classpathentry kind="lib" path="build/ivy/lib/default/ST4-@ST4.version@.jar"/>
   <classpathentry kind="lib" path="build/ivy/lib/default/snappy-@snappy.version@.jar"/>
+  <classpathentry kind="lib" path="build/beeline/hive-beeline-@HIVE_VERSION@.jar"/>
   <classpathentry kind="lib" path="build/ivy/lib/default/tempus-fugit-@tempus-fugit.version@.jar"/>
   <classpathentry kind="lib" path="build/builtins/hive-builtins-@HIVE_VERSION@.jar"/>
   <classpathentry kind="src" path="build/contrib/test/src"/>
   <classpathentry kind="src" path="build/metastore/gen/antlr/gen-java"/>
+  <classpathentry kind="lib" path="build/testutils/hive-testutils-@HIVE_VERSION@.jar"/>
   <classpathentry kind="src" path="build/ql/test/src"/>
   <classpathentry kind="src" path="build/ql/gen/antlr/gen-java"/>
   <classpathentry kind="src" path="beeline/src/java"/>

Modified: hive/branches/ptf-windowing/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDe.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDe.java?rev=1462670&r1=1462669&r2=1462670&view=diff
==============================================================================
--- hive/branches/ptf-windowing/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDe.java (original)
+++ hive/branches/ptf-windowing/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDe.java Fri Mar 29 22:58:43 2013
@@ -31,6 +31,7 @@ import org.apache.hadoop.hbase.client.Pu
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.hive.serde2.AbstractSerDe;
 import org.apache.hadoop.hive.serde2.ByteStream;
 import org.apache.hadoop.hive.serde2.SerDe;
 import org.apache.hadoop.hive.serde2.SerDeException;
@@ -58,7 +59,7 @@ import org.apache.hadoop.io.Writable;
  * HBaseSerDe can be used to serialize object into an HBase table and
  * deserialize objects from an HBase table.
  */
-public class HBaseSerDe implements SerDe {
+public class HBaseSerDe extends AbstractSerDe {
 
   public static final String HBASE_COLUMNS_MAPPING = "hbase.columns.mapping";
   public static final String HBASE_TABLE_NAME = "hbase.table.name";
@@ -546,10 +547,11 @@ public class HBaseSerDe implements SerDe
         throw new SerDeException("HBase row key cannot be NULL");
       }
 
-      if(putTimestamp >= 0)
+      if(putTimestamp >= 0) {
         put = new Put(key,putTimestamp);
-      else
+      } else {
         put = new Put(key);
+      }
 
       // Serialize each field
       for (int i = 0; i < fields.size(); i++) {

Modified: hive/branches/ptf-windowing/hcatalog/build.xml
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/hcatalog/build.xml?rev=1462670&r1=1462669&r2=1462670&view=diff
==============================================================================
--- hive/branches/ptf-windowing/hcatalog/build.xml (original)
+++ hive/branches/ptf-windowing/hcatalog/build.xml Fri Mar 29 22:58:43 2013
@@ -26,5 +26,6 @@
   -->
 
   <!-- temporarily override the test target to prevent build failures -->
+  <!-- small change -->
   <target name="test"/>
 </project>

Propchange: hive/branches/ptf-windowing/hcatalog/historical/branches/branch-0.4/
------------------------------------------------------------------------------
--- svn:externals (added)
+++ svn:externals Fri Mar 29 22:58:43 2013
@@ -0,0 +1 @@
+src/test/e2e/harness http://svn.apache.org/repos/asf/pig/trunk/test/e2e/harness

Propchange: hive/branches/ptf-windowing/hcatalog/historical/branches/branch-0.4/
------------------------------------------------------------------------------
--- svn:mergeinfo (added)
+++ svn:mergeinfo Fri Mar 29 22:58:43 2013
@@ -0,0 +1,2 @@
+/hive/trunk/hcatalog/historical/branches/branch-0.4:1458525-1462327
+/incubator/hcatalog/trunk:1296568,1299080,1304536,1307029,1325864,1327167,1327285,1329321,1332736,1332807,1332858,1336456,1338996,1340149,1365770,1373743,1388158,1392116

Propchange: hive/branches/ptf-windowing/hcatalog/historical/branches/branch-0.4.0-rc4/
------------------------------------------------------------------------------
--- svn:externals (added)
+++ svn:externals Fri Mar 29 22:58:43 2013
@@ -0,0 +1 @@
+src/test/e2e/harness http://svn.apache.org/repos/asf/pig/trunk/test/e2e/harness

Propchange: hive/branches/ptf-windowing/hcatalog/historical/branches/branch-0.4.0-rc4/
------------------------------------------------------------------------------
--- svn:mergeinfo (added)
+++ svn:mergeinfo Fri Mar 29 22:58:43 2013
@@ -0,0 +1,2 @@
+/hive/trunk/hcatalog/historical/branches/branch-0.4.0-rc4:1458525-1462327
+/incubator/hcatalog/trunk:1296568,1299080,1304536,1307029

Propchange: hive/branches/ptf-windowing/hcatalog/historical/branches/branch-0.4.0-rc4/hive/
------------------------------------------------------------------------------
--- svn:externals (added)
+++ svn:externals Fri Mar 29 22:58:43 2013
@@ -0,0 +1 @@
+

Propchange: hive/branches/ptf-windowing/hcatalog/historical/branches/branch-0.4/hive/
------------------------------------------------------------------------------
--- svn:externals (added)
+++ svn:externals Fri Mar 29 22:58:43 2013
@@ -0,0 +1 @@
+

Propchange: hive/branches/ptf-windowing/hcatalog/historical/branches/branch-0.5/
------------------------------------------------------------------------------
--- svn:externals (added)
+++ svn:externals Fri Mar 29 22:58:43 2013
@@ -0,0 +1 @@
+src/test/e2e/harness http://svn.apache.org/repos/asf/pig/trunk/test/e2e/harness

Propchange: hive/branches/ptf-windowing/hcatalog/historical/site/author/src/documentation/
------------------------------------------------------------------------------
--- svn:externals (added)
+++ svn:externals Fri Mar 29 22:58:43 2013
@@ -0,0 +1 @@
+skins http://svn.apache.org/repos/asf/hadoop/common/site/main/author/src/documentation/skins

Propchange: hive/branches/ptf-windowing/hcatalog/historical/trunk/
------------------------------------------------------------------------------
--- svn:externals (added)
+++ svn:externals Fri Mar 29 22:58:43 2013
@@ -0,0 +1 @@
+src/test/e2e/harness http://svn.apache.org/repos/asf/pig/trunk/test/e2e/harness

Modified: hive/branches/ptf-windowing/ivy/libraries.properties
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ivy/libraries.properties?rev=1462670&r1=1462669&r2=1462670&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ivy/libraries.properties (original)
+++ hive/branches/ptf-windowing/ivy/libraries.properties Fri Mar 29 22:58:43 2013
@@ -65,9 +65,9 @@ sqlline.version=1_0_2
 sqlline.branch=1.0.2
 slf4j-api.version=1.6.1
 slf4j-log4j12.version=1.6.1
+ST4.version=4.0.4
 tempus-fugit.version=1.1
 snappy.version=0.2
 velocity.version=1.5
 zookeeper.version=3.4.3
 javolution.version=5.5.1
-stringtemplate.version=3.2.1

Modified: hive/branches/ptf-windowing/jdbc/src/test/org/apache/hive/jdbc/TestJdbcDriver2.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/jdbc/src/test/org/apache/hive/jdbc/TestJdbcDriver2.java?rev=1462670&r1=1462669&r2=1462670&view=diff
==============================================================================
--- hive/branches/ptf-windowing/jdbc/src/test/org/apache/hive/jdbc/TestJdbcDriver2.java (original)
+++ hive/branches/ptf-windowing/jdbc/src/test/org/apache/hive/jdbc/TestJdbcDriver2.java Fri Mar 29 22:58:43 2013
@@ -811,17 +811,19 @@ public class TestJdbcDriver2 extends Tes
 
     ResultSet res = stmt.executeQuery("describe " + tableName);
 
+    res.next(); // skip header 1
+    res.next(); // skip header 2
+
     res.next();
-    assertEquals("Column name 'under_col' not found", "under_col", res.getString(1));
+    assertEquals("Column name 'under_col' not found", "under_col", res.getString(1).trim());
     assertEquals("Column type 'under_col' for column under_col not found", "int", res
-        .getString(2));
+        .getString(2).trim());
     res.next();
-    assertEquals("Column name 'value' not found", "value", res.getString(1));
+    assertEquals("Column name 'value' not found", "value", res.getString(1).trim());
     assertEquals("Column type 'string' for column key not found", "string", res
-        .getString(2));
+        .getString(2).trim());
 
     assertFalse("More results found than expected", res.next());
-
   }
 
   public void testDatabaseMetaData() throws SQLException {

Modified: hive/branches/ptf-windowing/metastore/ivy.xml
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/metastore/ivy.xml?rev=1462670&r1=1462669&r2=1462670&view=diff
==============================================================================
--- hive/branches/ptf-windowing/metastore/ivy.xml (original)
+++ hive/branches/ptf-windowing/metastore/ivy.xml Fri Mar 29 22:58:43 2013
@@ -26,10 +26,11 @@
     <include file="${ivy.conf.dir}/common-configurations.xml"/>
   </configurations>
   <dependencies>
+    <dependency org="org.antlr" name="antlr" rev="${antlr.version}" transitive="false"/>
+    <dependency org="org.antlr" name="antlr-runtime" rev="${antlr-runtime.version}" transitive="false"/>
+    <dependency org="org.antlr" name="ST4" rev="${ST4.version}" transitive="false"/><!-- manually added (antlr dep), bad POM -->
     <dependency org="org.apache.hive" name="hive-serde" rev="${version}"
                 conf="compile->default" />
-    <dependency org="org.antlr" name="antlr" rev="${antlr.version}"/>
-    <dependency org="org.antlr" name="antlr-runtime" rev="${antlr-runtime.version}"/>
     <dependency org="commons-dbcp" name="commons-dbcp" rev="${commons-dbcp.version}">
       <exclude module="commons-pool" />
       <exclude org="org.apache.geronimo.specs" module="geronimo-jta_1.1_spec"/>

Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java?rev=1462670&r1=1462669&r2=1462670&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java (original)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java Fri Mar 29 22:58:43 2013
@@ -236,8 +236,8 @@ public enum ErrorMsg {
     "Fix the metadata or don't use bucketed mapjoin, by setting " +
     "hive.enforce.bucketmapjoin to false."),
 
-  JOINNODE_OUTERJOIN_MORETHAN_8(10142, "Single join node containing outer join(s) " +
-      "cannot have more than 8 aliases"),
+  JOINNODE_OUTERJOIN_MORETHAN_32(10142, "Single join node containing outer join(s) " +
+      "cannot have more than 32 aliases"),
 
   INVALID_JDO_FILTER_EXPRESSION(10043, "Invalid expression for JDO filter"),
 
@@ -328,6 +328,17 @@ public enum ErrorMsg {
   ANALYZE_TABLE_NOSCAN_NON_NATIVE(10228, "ANALYZE TABLE NOSCAN cannot be used for "
       + "a non-native table"),
 
+  ANALYZE_TABLE_PARTIALSCAN_NON_NATIVE(10229, "ANALYZE TABLE PARTIALSCAN cannot be used for "
+      + "a non-native table"),
+  ANALYZE_TABLE_PARTIALSCAN_NON_RCFILE(10230, "ANALYZE TABLE PARTIALSCAN doesn't "
+      + "support non-RCfile. "),
+  ANALYZE_TABLE_PARTIALSCAN_EXTERNAL_TABLE(10231, "ANALYZE TABLE PARTIALSCAN "
+      + "doesn't support external table: "),
+  ANALYZE_TABLE_PARTIALSCAN_AGGKEY(10232, "Analyze partialscan command "
+            + "fails to construct aggregation for the partition "),
+  ANALYZE_TABLE_PARTIALSCAN_AUTOGATHER(10233, "Analyze partialscan is not allowed " +
+            "if hive.stats.autogather is set to false"),
+
   SCRIPT_INIT_ERROR(20000, "Unable to initialize custom script."),
   SCRIPT_IO_ERROR(20001, "An error occurred while reading or writing to your custom script. "
       + "It may have crashed with an error."),

Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java?rev=1462670&r1=1462669&r2=1462670&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java (original)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java Fri Mar 29 22:58:43 2013
@@ -45,6 +45,7 @@ public class QueryProperties {
 
   boolean hasDistributeBy = false;
   boolean hasClusterBy = false;
+  boolean mapJoinRemoved = false;
 
   public boolean hasJoin() {
     return hasJoin;
@@ -125,4 +126,12 @@ public class QueryProperties {
   public void setHasWindowing(boolean hasWindowing) {
     this.hasWindowing = hasWindowing;
   }
+
+  public boolean isMapJoinRemoved() {
+    return mapJoinRemoved;
+  }
+
+  public void setMapJoinRemoved(boolean mapJoinRemoved) {
+    this.mapJoinRemoved = mapJoinRemoved;
+  }
 }

Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java?rev=1462670&r1=1462669&r2=1462670&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java (original)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java Fri Mar 29 22:58:43 2013
@@ -21,7 +21,6 @@ package org.apache.hadoop.hive.ql.exec;
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -36,7 +35,7 @@ import org.apache.hadoop.hive.ql.metadat
 import org.apache.hadoop.hive.ql.plan.JoinCondDesc;
 import org.apache.hadoop.hive.ql.plan.JoinDesc;
 import org.apache.hadoop.hive.ql.plan.TableDesc;
-import org.apache.hadoop.hive.serde2.io.ByteWritable;
+import org.apache.hadoop.hive.serde2.io.ShortWritable;
 import org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
@@ -52,40 +51,6 @@ public abstract class CommonJoinOperator
   protected static final Log LOG = LogFactory.getLog(CommonJoinOperator.class
       .getName());
 
-  /**
-   * IntermediateObject.
-   *
-   */
-  public static class IntermediateObject {
-    ArrayList<Object>[] objs;
-    int curSize;
-
-    public IntermediateObject(ArrayList<Object>[] objs, int curSize) {
-      this.objs = objs;
-      this.curSize = curSize;
-    }
-
-    public ArrayList<Object>[] getObjs() {
-      return objs;
-    }
-
-    public int getCurSize() {
-      return curSize;
-    }
-
-    public void pushObj(ArrayList<Object> newObj) {
-      objs[curSize++] = newObj;
-    }
-
-    public void popObj() {
-      curSize--;
-    }
-
-    public Object topObj() {
-      return objs[curSize - 1];
-    }
-  }
-
   protected transient int numAliases; // number of aliases
   /**
    * The expressions for join inputs.
@@ -97,7 +62,7 @@ public abstract class CommonJoinOperator
    */
   protected transient List<ExprNodeEvaluator>[] joinFilters;
 
-  protected transient int[][] filterMap;
+  protected transient int[][] filterMaps;
 
   /**
    * The ObjectInspectors for the join inputs.
@@ -123,14 +88,13 @@ public abstract class CommonJoinOperator
   protected transient boolean[] nullsafes;
 
   public transient boolean noOuterJoin;
-  protected transient Object[] dummyObj; // for outer joins, contains the
-  // potential nulls for the concerned
-  // aliases
-  protected transient RowContainer<ArrayList<Object>>[] dummyObjVectors; // empty
-  // rows
-  // for
-  // each
-  // table
+
+  // for outer joins, contains the potential nulls for the concerned aliases
+  protected transient ArrayList<Object>[] dummyObj;
+
+  // empty rows for each table
+  protected transient RowContainer<ArrayList<Object>>[] dummyObjVectors;
+
   protected transient int totalSz; // total size of the composite object
 
   // keys are the column names. basically this maps the position of the column
@@ -264,7 +228,7 @@ public abstract class CommonJoinOperator
     joinValuesStandardObjectInspectors = JoinUtil.getStandardObjectInspectors(
         joinValuesObjectInspectors,NOTSKIPBIGTABLE, tagLen);
 
-    filterMap = conf.getFilterMap();
+    filterMaps = conf.getFilterMap();
 
     if (noOuterJoin) {
       rowContainerStandardObjectInspectors = joinValuesStandardObjectInspectors;
@@ -273,19 +237,16 @@ public abstract class CommonJoinOperator
       for (Byte alias : order) {
         ArrayList<ObjectInspector> rcOIs = new ArrayList<ObjectInspector>();
         rcOIs.addAll(joinValuesObjectInspectors[alias]);
-        // for each alias, add object inspector for boolean as the last element
+        // for each alias, add object inspector for short as the last element
         rcOIs.add(
-            PrimitiveObjectInspectorFactory.writableByteObjectInspector);
+            PrimitiveObjectInspectorFactory.writableShortObjectInspector);
         rowContainerObjectInspectors[alias] = rcOIs;
       }
       rowContainerStandardObjectInspectors =
         JoinUtil.getStandardObjectInspectors(rowContainerObjectInspectors,NOTSKIPBIGTABLE, tagLen);
     }
 
-
-
-
-    dummyObj = new Object[numAliases];
+    dummyObj = new ArrayList[numAliases];
     dummyObjVectors = new RowContainer[numAliases];
 
     joinEmitInterval = HiveConf.getIntVar(hconf,
@@ -309,7 +270,7 @@ public abstract class CommonJoinOperator
         // add whether the row is filtered or not
         // this value does not matter for the dummyObj
         // because the join values are already null
-        nr.add(new ByteWritable());
+        nr.add(new ShortWritable());
       }
       dummyObj[pos] = nr;
       // there should be only 1 dummy object in the RowContainer
@@ -317,20 +278,37 @@ public abstract class CommonJoinOperator
           rowContainerStandardObjectInspectors[pos],
           alias, 1, spillTableDesc, conf, !hasFilter(pos), reporter);
 
-      values.add((ArrayList<Object>) dummyObj[pos]);
+      values.add(dummyObj[pos]);
       dummyObjVectors[pos] = values;
 
       // if serde is null, the input doesn't need to be spilled out
       // e.g., the output columns does not contains the input table
       RowContainer rc = JoinUtil.getRowContainer(hconf,
           rowContainerStandardObjectInspectors[pos],
-          alias, joinCacheSize,spillTableDesc, conf, !hasFilter(pos), reporter);
+          alias, joinCacheSize, spillTableDesc, conf, !hasFilter(pos), reporter);
       storage[pos] = rc;
 
       pos++;
     }
 
     forwardCache = new Object[totalSz];
+    aliasFilterTags = new short[numAliases];
+    Arrays.fill(aliasFilterTags, (byte)0xff);
+
+    filterTags = new short[numAliases];
+    skipVectors = new boolean[numAliases][];
+    for(int i = 0; i < skipVectors.length; i++) {
+      skipVectors[i] = new boolean[i + 1];
+    }
+    intermediate = new List[numAliases];
+
+    offsets = new int[numAliases + 1];
+    int sum = 0;
+    for (int i = 0; i < numAliases; i++) {
+      offsets[i] = sum;
+      sum += joinValues[order[i]].size();
+    }
+    offsets[numAliases] = sum;
 
     outputObjInspector = getJoinOutputObjectInspector(order,
         joinValuesStandardObjectInspectors, conf);
@@ -341,16 +319,11 @@ public abstract class CommonJoinOperator
       }
     }
 
-    LOG.info("JOIN "
-        + outputObjInspector.getTypeName()
-        + " totalsz = " + totalSz);
-
+    LOG.info("JOIN " + outputObjInspector.getTypeName() + " totalsz = " + totalSz);
   }
 
+  transient boolean newGroupStarted = false;
 
-
-
-transient boolean newGroupStarted = false;
   @Override
   public void startGroup() throws HiveException {
     LOG.trace("Join: Starting new group");
@@ -370,353 +343,267 @@ transient boolean newGroupStarted = fals
   }
 
   protected transient Byte alias;
+  protected transient Object[] forwardCache;
 
-  transient Object[] forwardCache;
+  // pre-calculated offset values for each alias
+  protected transient int[] offsets;
 
-  private void createForwardJoinObject(IntermediateObject intObj,
-      boolean[] nullsArr) throws HiveException {
-    int p = 0;
+  // a array of bitvectors where each entry denotes whether the element is to
+  // be used or not (whether it is null or not). The size of the bitvector is
+  // same as the number of inputs(aliases) under consideration currently.
+  // When all inputs are accounted for, the output is forwarded appropriately.
+  protected transient boolean[][] skipVectors;
+
+  // caches objects before constructing forward cache
+  protected transient List[] intermediate;
+
+  // filter tags for objects
+  protected transient short[] filterTags;
+
+  // ANDed value of all filter tags in current join group
+  // if any of values passes on outer join alias (which makes zero for the tag alias),
+  // it means there exists a pair for it, and no need to check outer join (just do inner join)
+  //
+  // for example, with table a, b something like,
+  //   a, b = 100, 10 | 100, 20 | 100, 30
+  //
+  // the query "a FOJ b ON a.k=b.k AND a.v>0 AND b.v>20" makes values with tag
+  //
+  //   a = 100, 10, 00000010 | 100, 20, 00000010 | 100, 30, 00000010 : 0/1 for 'b' (alias 1)
+  //   b = 100, 10, 00000001 | 100, 20, 00000001 | 100, 30, 00000000 : 0/1 for 'a' (alias 0)
+  //
+  // which makes aliasFilterTags for a = 00000010, for b = 00000000
+  //
+  // for LO, b = 0000000(0) means there is a pair object(s) in 'b' (has no 'a'-null case)
+  // for RO, a = 000000(1)0 means there is no pair object in 'a' (has null-'b' case)
+  //
+  // result : 100, 10 + 100, 30 | 100, 20 + 100, 30 | 100, 30 + 100, 30 |
+  //          N       + 100, 10 | N       + 100, 20
+  //
+  protected transient short[] aliasFilterTags;
+
+  // all evaluation should be processed here for valid aliasFilterTags
+  //
+  // for MapJoin, filter tag is pre-calculated in MapredLocalTask and stored with value.
+  // when reading the hashtable, MapJoinObjectValue calcuates alias filter and provide it to join
+  protected ArrayList<Object> getFilteredValue(byte alias, Object row) throws HiveException {
+    boolean hasFilter = hasFilter(alias);
+    ArrayList<Object> nr = JoinUtil.computeValues(row, joinValues[alias],
+        joinValuesObjectInspectors[alias], hasFilter);
+    if (hasFilter) {
+      short filterTag = JoinUtil.isFiltered(row, joinFilters[alias],
+          joinFilterObjectInspectors[alias], filterMaps[alias]);
+      nr.add(new ShortWritable(filterTag));
+      aliasFilterTags[alias] &= filterTag;
+    }
+    return nr;
+  }
+
+  // fill forwardCache with skipvector
+  private void createForwardJoinObject(boolean[] skip) throws HiveException {
+    Arrays.fill(forwardCache, null);
+
+    boolean forward = false;
     for (int i = 0; i < numAliases; i++) {
-      Byte alias = order[i];
-      int sz = joinValues[alias].size();
-      if (nullsArr[i]) {
-        for (int j = 0; j < sz; j++) {
-          forwardCache[p++] = null;
-        }
-      } else {
-        ArrayList<Object> obj = intObj.getObjs()[i];
-        for (int j = 0; j < sz; j++) {
-          forwardCache[p++] = obj.get(j);
+      if (!skip[i]) {
+        for (int j = offsets[i]; j < offsets[i + 1]; j++) {
+          forwardCache[j] = intermediate[i].get(j - offsets[i]);
         }
+        forward = true;
       }
     }
-
-    forward(forwardCache, outputObjInspector);
-    countAfterReport = 0;
-  }
-
-  private void copyOldArray(boolean[] src, boolean[] dest) {
-    for (int i = 0; i < src.length; i++) {
-      dest[i] = src[i];
+    if (forward) {
+      forward(forwardCache, null);
+      countAfterReport = 0;
     }
   }
 
-  private ArrayList<boolean[]> joinObjectsInnerJoin(
-      ArrayList<boolean[]> resNulls, ArrayList<boolean[]> inputNulls,
-      ArrayList<Object> newObj, IntermediateObject intObj, int left,
-      boolean newObjNull) {
-    if (newObjNull) {
-      return resNulls;
-    }
-    Iterator<boolean[]> nullsIter = inputNulls.iterator();
-    while (nullsIter.hasNext()) {
-      boolean[] oldNulls = nullsIter.next();
-      boolean oldObjNull = oldNulls[left];
-      if (!oldObjNull) {
-        boolean[] newNulls = new boolean[intObj.getCurSize()];
-        copyOldArray(oldNulls, newNulls);
-        newNulls[oldNulls.length] = false;
-        resNulls.add(newNulls);
-      }
-    }
-    return resNulls;
-  }
+  // entry point (aliasNum = 0)
+  private void genJoinObject() throws HiveException {
+    boolean rightFirst = true;
+    boolean hasFilter = hasFilter(order[0]);
+    AbstractRowContainer<ArrayList<Object>> aliasRes = storage[order[0]];
+    for (List<Object> rightObj = aliasRes.first(); rightObj != null; rightObj = aliasRes.next()) {
+      boolean rightNull = rightObj == dummyObj[0];
+      if (hasFilter) {
+        filterTags[0] = getFilterTag(rightObj);
+      }
+      skipVectors[0][0] = rightNull;
+      intermediate[0] = rightObj;
 
-  /**
-   * Implement semi join operator.
-   */
-  private ArrayList<boolean[]> joinObjectsLeftSemiJoin(
-      ArrayList<boolean[]> resNulls, ArrayList<boolean[]> inputNulls,
-      ArrayList<Object> newObj, IntermediateObject intObj, int left,
-      boolean newObjNull) {
-    if (newObjNull) {
-      return resNulls;
-    }
-    Iterator<boolean[]> nullsIter = inputNulls.iterator();
-    while (nullsIter.hasNext()) {
-      boolean[] oldNulls = nullsIter.next();
-      boolean oldObjNull = oldNulls[left];
-      if (!oldObjNull) {
-        boolean[] newNulls = new boolean[intObj.getCurSize()];
-        copyOldArray(oldNulls, newNulls);
-        newNulls[oldNulls.length] = false;
-        resNulls.add(newNulls);
-      }
-    }
-    return resNulls;
-  }
-
-  private ArrayList<boolean[]> joinObjectsLeftOuterJoin(
-      ArrayList<boolean[]> resNulls, ArrayList<boolean[]> inputNulls,
-      ArrayList<Object> newObj, IntermediateObject intObj, int left, int right,
-      boolean newObjNull) {
-    // newObj is null if is already null or
-    // if the row corresponding to the left alias does not pass through filter
-    newObjNull |= isLeftFiltered(left, right, intObj.getObjs()[left]);
-
-    Iterator<boolean[]> nullsIter = inputNulls.iterator();
-    while (nullsIter.hasNext()) {
-      boolean[] oldNulls = nullsIter.next();
-      boolean oldObjNull = oldNulls[left];
-      boolean[] newNulls = new boolean[intObj.getCurSize()];
-      copyOldArray(oldNulls, newNulls);
-      if (oldObjNull) {
-        newNulls[oldNulls.length] = true;
-      } else {
-        newNulls[oldNulls.length] = newObjNull;
-      }
-      resNulls.add(newNulls);
+      genObject(1, rightFirst, rightNull);
+      rightFirst = false;
     }
-    return resNulls;
   }
 
-  private ArrayList<boolean[]> joinObjectsRightOuterJoin(
-      ArrayList<boolean[]> resNulls, ArrayList<boolean[]> inputNulls,
-      ArrayList<Object> newObj, IntermediateObject intObj, int left, int right,
-      boolean newObjNull, boolean firstRow) {
-    if (newObjNull) {
-      return resNulls;
-    }
-
-    if (inputNulls.isEmpty() && firstRow) {
-      boolean[] newNulls = new boolean[intObj.getCurSize()];
-      for (int i = 0; i < intObj.getCurSize() - 1; i++) {
-        newNulls[i] = true;
-      }
-      newNulls[intObj.getCurSize() - 1] = newObjNull;
-      resNulls.add(newNulls);
-      return resNulls;
-    }
+  // creates objects in recursive manner
+  private void genObject(int aliasNum, boolean allLeftFirst, boolean allLeftNull)
+      throws HiveException {
+    if (aliasNum < numAliases) {
 
-    boolean allOldObjsNull = firstRow;
+      boolean[] skip = skipVectors[aliasNum];
+      boolean[] prevSkip = skipVectors[aliasNum - 1];
 
-    Iterator<boolean[]> nullsIter = inputNulls.iterator();
-    while (nullsIter.hasNext()) {
-      boolean[] oldNulls = nullsIter.next();
-      if (!oldNulls[left]) {
-        allOldObjsNull = false;
-        break;
-      }
-    }
+      JoinCondDesc joinCond = condn[aliasNum - 1];
+      int type = joinCond.getType();
+      int left = joinCond.getLeft();
+      int right = joinCond.getRight();
 
-    // if the row does not pass through filter, all old Objects are null
-    if (isRightFiltered(left, right, newObj)) {
-      allOldObjsNull = true;
-    }
-    nullsIter = inputNulls.iterator();
-    while (nullsIter.hasNext()) {
-      boolean[] oldNulls = nullsIter.next();
-      boolean oldObjNull = oldNulls[left] || allOldObjsNull;
+      // search for match in the rhs table
+      AbstractRowContainer<ArrayList<Object>> aliasRes = storage[order[aliasNum]];
 
-      if (!oldObjNull) {
-        boolean[] newNulls = new boolean[intObj.getCurSize()];
-        copyOldArray(oldNulls, newNulls);
-        newNulls[oldNulls.length] = newObjNull;
-        resNulls.add(newNulls);
-      } else if (allOldObjsNull) {
-        boolean[] newNulls = new boolean[intObj.getCurSize()];
-        for (int i = 0; i < intObj.getCurSize() - 1; i++) {
-          newNulls[i] = true;
+      boolean done = false;
+      boolean loopAgain = false;
+      boolean tryLOForFO = type == JoinDesc.FULL_OUTER_JOIN;
+
+      boolean rightFirst = true;
+      for (List<Object> rightObj = aliasRes.first(); !done && rightObj != null;
+           rightObj = loopAgain ? rightObj : aliasRes.next(), rightFirst = loopAgain = false) {
+        System.arraycopy(prevSkip, 0, skip, 0, prevSkip.length);
+
+        boolean rightNull = rightObj == dummyObj[aliasNum];
+        if (hasFilter(order[aliasNum])) {
+          filterTags[aliasNum] = getFilterTag(rightObj);
         }
-        newNulls[oldNulls.length] = newObjNull;
-        resNulls.add(newNulls);
-        return resNulls;
-      }
-    }
-    return resNulls;
-  }
-
-  private ArrayList<boolean[]> joinObjectsFullOuterJoin(
-      ArrayList<boolean[]> resNulls, ArrayList<boolean[]> inputNulls,
-      ArrayList<Object> newObj, IntermediateObject intObj, int left, int right,
-      boolean newObjNull, boolean firstRow) {
-    if (newObjNull) {
-      Iterator<boolean[]> nullsIter = inputNulls.iterator();
-      while (nullsIter.hasNext()) {
-        boolean[] oldNulls = nullsIter.next();
-        boolean[] newNulls = new boolean[intObj.getCurSize()];
-        copyOldArray(oldNulls, newNulls);
-        newNulls[oldNulls.length] = newObjNull;
-        resNulls.add(newNulls);
-      }
-      return resNulls;
-    }
-
-    if (inputNulls.isEmpty() && firstRow) {
-      boolean[] newNulls = new boolean[intObj.getCurSize()];
-      for (int i = 0; i < intObj.getCurSize() - 1; i++) {
-        newNulls[i] = true;
-      }
-      newNulls[intObj.getCurSize() - 1] = newObjNull;
-      resNulls.add(newNulls);
-      return resNulls;
-    }
-
-    boolean allOldObjsNull = firstRow;
-
-    Iterator<boolean[]> nullsIter = inputNulls.iterator();
-    while (nullsIter.hasNext()) {
-      boolean[] oldNulls = nullsIter.next();
-      if (!oldNulls[left]) {
-        allOldObjsNull = false;
-        break;
-      }
-    }
-
-    // if the row does not pass through filter, all old Objects are null
-    if (isRightFiltered(left, right, newObj)) {
-      allOldObjsNull = true;
-    }
-    boolean rhsPreserved = false;
-
-    nullsIter = inputNulls.iterator();
-    while (nullsIter.hasNext()) {
-      boolean[] oldNulls = nullsIter.next();
-      // old obj is null even if the row corresponding to the left alias
-      // does not pass through filter
-      boolean oldObjNull = oldNulls[left] || allOldObjsNull
-          || isLeftFiltered(left, right, intObj.getObjs()[left]);
-      if (!oldObjNull) {
-        boolean[] newNulls = new boolean[intObj.getCurSize()];
-        copyOldArray(oldNulls, newNulls);
-        newNulls[oldNulls.length] = newObjNull;
-        resNulls.add(newNulls);
-      } else if (oldObjNull) {
-        boolean[] newNulls = new boolean[intObj.getCurSize()];
-        copyOldArray(oldNulls, newNulls);
-        newNulls[oldNulls.length] = true;
-        resNulls.add(newNulls);
-
-        if (allOldObjsNull && !rhsPreserved) {
-          newNulls = new boolean[intObj.getCurSize()];
-          for (int i = 0; i < oldNulls.length; i++) {
-            newNulls[i] = true;
+        skip[right] = rightNull;
+
+        if (type == JoinDesc.INNER_JOIN) {
+          innerJoin(skip, left, right);
+        } else if (type == JoinDesc.LEFT_SEMI_JOIN) {
+          if (innerJoin(skip, left, right)) {
+            // if left-semi-join found a match, skipping the rest of the rows in the
+            // rhs table of the semijoin
+            done = true;
+          }
+        } else if (type == JoinDesc.LEFT_OUTER_JOIN ||
+            (type == JoinDesc.FULL_OUTER_JOIN && rightNull)) {
+          int result = leftOuterJoin(skip, left, right);
+          if (result < 0) {
+            continue;
+          }
+          done = result > 0;
+        } else if (type == JoinDesc.RIGHT_OUTER_JOIN ||
+            (type == JoinDesc.FULL_OUTER_JOIN && allLeftNull)) {
+          if (allLeftFirst && !rightOuterJoin(skip, left, right) ||
+            !allLeftFirst && !innerJoin(skip, left, right)) {
+            continue;
+          }
+        } else if (type == JoinDesc.FULL_OUTER_JOIN) {
+          if (tryLOForFO && leftOuterJoin(skip, left, right) > 0) {
+            loopAgain = allLeftFirst;
+            done = !loopAgain;
+            tryLOForFO = false;
+          } else if (allLeftFirst && !rightOuterJoin(skip, left, right) ||
+            !allLeftFirst && !innerJoin(skip, left, right)) {
+            continue;
           }
-          newNulls[oldNulls.length] = false;
-          resNulls.add(newNulls);
-          rhsPreserved = true;
         }
+        intermediate[aliasNum] = rightObj;
+
+        // recursively call the join the other rhs tables
+        genObject(aliasNum + 1, allLeftFirst && rightFirst, allLeftNull && rightNull);
       }
+    } else if (!allLeftNull) {
+      createForwardJoinObject(skipVectors[numAliases - 1]);
     }
-    return resNulls;
   }
 
-  /*
-   * The new input is added to the list of existing inputs. Each entry in the
-   * array of inputNulls denotes the entries in the intermediate object to be
-   * used. The intermediate object is augmented with the new object, and list of
-   * nulls is changed appropriately. The list will contain all non-nulls for a
-   * inner join. The outer joins are processed appropriately.
-   */
-  private ArrayList<boolean[]> joinObjects(ArrayList<boolean[]> inputNulls,
-      ArrayList<Object> newObj, IntermediateObject intObj, int joinPos,
-      boolean firstRow) {
-    ArrayList<boolean[]> resNulls = new ArrayList<boolean[]>();
-    boolean newObjNull = newObj == dummyObj[joinPos] ? true : false;
-    if (joinPos == 0) {
-      if (newObjNull) {
-        return null;
-      }
-      boolean[] nulls = new boolean[1];
-      nulls[0] = newObjNull;
-      resNulls.add(nulls);
-      return resNulls;
-    }
-
-    int left = condn[joinPos - 1].getLeft();
-    int right = condn[joinPos - 1].getRight();
-    int type = condn[joinPos - 1].getType();
-
-    // process all nulls for RIGHT and FULL OUTER JOINS
-    if (((type == JoinDesc.RIGHT_OUTER_JOIN) || (type == JoinDesc.FULL_OUTER_JOIN))
-        && !newObjNull && (inputNulls == null) && firstRow) {
-      boolean[] newNulls = new boolean[intObj.getCurSize()];
-      for (int i = 0; i < newNulls.length - 1; i++) {
-        newNulls[i] = true;
-      }
-      newNulls[newNulls.length - 1] = false;
-      resNulls.add(newNulls);
-      return resNulls;
-    }
-
-    if (inputNulls == null) {
-      return null;
-    }
-
-    if (type == JoinDesc.INNER_JOIN) {
-      return joinObjectsInnerJoin(resNulls, inputNulls, newObj, intObj, left,
-          newObjNull);
-    } else if (type == JoinDesc.LEFT_OUTER_JOIN) {
-      return joinObjectsLeftOuterJoin(resNulls, inputNulls, newObj, intObj,
-          left, right, newObjNull);
-    } else if (type == JoinDesc.RIGHT_OUTER_JOIN) {
-      return joinObjectsRightOuterJoin(resNulls, inputNulls, newObj, intObj,
-          left, right, newObjNull, firstRow);
-    } else if (type == JoinDesc.LEFT_SEMI_JOIN) {
-      return joinObjectsLeftSemiJoin(resNulls, inputNulls, newObj, intObj,
-          left, newObjNull);
-    }
-
-    assert (type == JoinDesc.FULL_OUTER_JOIN);
-    return joinObjectsFullOuterJoin(resNulls, inputNulls, newObj, intObj, left, right,
-        newObjNull, firstRow);
-  }
-
-  /*
-   * genObject is a recursive function. For the inputs, a array of bitvectors is
-   * maintained (inputNulls) where each entry denotes whether the element is to
-   * be used or not (whether it is null or not). The size of the bitvector is
-   * same as the number of inputs under consideration currently. When all inputs
-   * are accounted for, the output is forwarded appropriately.
-   */
-  private void genObject(ArrayList<boolean[]> inputNulls, int aliasNum,
-      IntermediateObject intObj, boolean firstRow) throws HiveException {
-    boolean childFirstRow = firstRow;
-    boolean skipping = false;
+  // inner join
+  private boolean innerJoin(boolean[] skip, int left, int right) {
+    if (!isInnerJoin(skip, left, right)) {
+      Arrays.fill(skip, true);
+      return false;
+    }
+    return true;
+  }
 
-    if (aliasNum < numAliases) {
+  // LO
+  //
+  // LEFT\RIGHT   skip  filtered   valid
+  // skip        --(1)     --(1)    --(1)
+  // filtered    +-(1)     +-(1)    +-(1)
+  // valid       +-(1)     +-(4*)   ++(2)
+  //
+  // * If right alias has any pair for left alias, continue (3)
+  // -1 for continue : has pair but not in this turn
+  //  0 for inner join (++) : join and continue LO
+  //  1 for left outer join (+-) : join and skip further LO
+  private int leftOuterJoin(boolean[] skip, int left, int right) {
+    if (skip[left] || skip[right] || !isLeftValid(left, right)) {
+      skip[right] = true;
+      return 1;   // case 1
+    }
+    if (isRightValid(left, right)) {
+      return 0;   // case 2
+    }
+    if (hasRightPairForLeft(left, right)) {
+      return -1;  // case 3
+    }
+    skip[right] = true;
+    return 1;     // case 4
+  }
 
-      // search for match in the rhs table
-      AbstractRowContainer<ArrayList<Object>> aliasRes = storage[order[aliasNum]];
+  // RO
+  //
+  // LEFT\RIGHT   skip  filtered   valid
+  // skip        --(1)     -+(1)   -+(1)
+  // filtered    --(1)     -+(1)   -+(4*)
+  // valid       --(1)     -+(1)   ++(2)
+  //
+  // * If left alias has any pair for right alias, continue (3)
+  // false for continue : has pair but not in this turn
+  private boolean rightOuterJoin(boolean[] skip, int left, int right) {
+    if (skip[left] || skip[right] || !isRightValid(left, right)) {
+      Arrays.fill(skip, 0, right, true);
+      return true;  // case 1
+    }
+    if (isLeftValid(left, right)) {
+      return true;  // case 2
+    }
+    if (hasLeftPairForRight(left, right)) {
+      return false; // case 3
+    }
+    Arrays.fill(skip, 0, right, true);
+    return true;    // case 4
+  }
 
-      for (ArrayList<Object> newObj = aliasRes.first(); newObj != null; newObj = aliasRes
-          .next()) {
+  // If left and right aliases are all valid, two values will be inner joined,
+  private boolean isInnerJoin(boolean[] skip, int left, int right) {
+    return !skip[left] && !skip[right] &&
+        isLeftValid(left, right) && isRightValid(left, right);
+  }
 
-        // check for skipping in case of left semi join
-        if (aliasNum > 0
-            && condn[aliasNum - 1].getType() == JoinDesc.LEFT_SEMI_JOIN
-            && newObj != dummyObj[aliasNum]) { // successful match
-          skipping = true;
-        }
+  // check if left is valid
+  private boolean isLeftValid(int left, int right) {
+    return !hasFilter(left) || !JoinUtil.isFiltered(filterTags[left], right);
+  }
+
+  // check if right is valid
+  private boolean isRightValid(int left, int right) {
+    return !hasFilter(right) || !JoinUtil.isFiltered(filterTags[right], left);
+  }
 
-        intObj.pushObj(newObj);
+  // check if any left pair exists for right objects
+  private boolean hasLeftPairForRight(int left, int right) {
+    return !JoinUtil.isFiltered(aliasFilterTags[left], right);
+  }
 
-        // execute the actual join algorithm
-        ArrayList<boolean[]> newNulls = joinObjects(inputNulls, newObj, intObj,
-            aliasNum, childFirstRow);
+  // check if any right pair exists for left objects
+  private boolean hasRightPairForLeft(int left, int right) {
+    return !JoinUtil.isFiltered(aliasFilterTags[right], left);
+  }
 
-        // recursively call the join the other rhs tables
-        genObject(newNulls, aliasNum + 1, intObj, firstRow);
+  private boolean hasAnyFiltered(int alias, List<Object> row) {
+    return row == dummyObj[alias] || hasFilter(alias) && JoinUtil.hasAnyFiltered(getFilterTag(row));
+  }
 
-        intObj.popObj();
-        firstRow = false;
+  protected final boolean hasFilter(int alias) {
+    return filterMaps != null && filterMaps[alias] != null;
+  }
 
-        // if left-semi-join found a match, skipping the rest of the rows in the
-        // rhs table of the semijoin
-        if (skipping) {
-          break;
-        }
-      }
-    } else {
-      if (inputNulls == null) {
-        return;
-      }
-      Iterator<boolean[]> nullsIter = inputNulls.iterator();
-      while (nullsIter.hasNext()) {
-        boolean[] nullsVec = nullsIter.next();
-        createForwardJoinObject(intObj, nullsVec);
-      }
-    }
+  // get tag value from object (last of list)
+  protected final short getFilterTag(List<Object> row) {
+    return ((ShortWritable) row.get(row.size() - 1)).get();
   }
 
   /**
@@ -783,7 +670,7 @@ transient boolean newGroupStarted = fals
         }
 
         if (alw.size() == 0) {
-          alw.add((ArrayList<Object>) dummyObj[i]);
+          alw.add(dummyObj[i]);
           hasNulls = true;
         } else if (condn[i].getPreserved()) {
           preserve = true;
@@ -821,7 +708,7 @@ transient boolean newGroupStarted = fals
         } else {
           if (alw.size() == 0) {
             hasEmpty = true;
-            alw.add((ArrayList<Object>) dummyObj[i]);
+            alw.add(dummyObj[i]);
           } else if (!hasEmpty && alw.size() == 1) {
             if (hasAnyFiltered(alias, alw.first())) {
               hasEmpty = true;
@@ -851,39 +738,11 @@ transient boolean newGroupStarted = fals
         LOG.trace("called genUniqueJoinObject");
       } else {
         LOG.trace("calling genObject");
-        genObject(null, 0, new IntermediateObject(new ArrayList[numAliases], 0),
-            true);
+        genJoinObject();
         LOG.trace("called genObject");
       }
     }
-  }
-
-  // returns filter result of left object by filters associated with right alias
-  private boolean isLeftFiltered(int left, int right, List<Object> leftObj) {
-    if (joinValues[order[left]].size() < leftObj.size()) {
-      ByteWritable filter = (ByteWritable) leftObj.get(leftObj.size() - 1);
-      return JoinUtil.isFiltered(filter.get(), right);
-    }
-    return false;
-  }
-
-  // returns filter result of right object by filters associated with left alias
-  private boolean isRightFiltered(int left, int right, List<Object> rightObj) {
-    if (joinValues[order[right]].size() < rightObj.size()) {
-      ByteWritable filter = (ByteWritable) rightObj.get(rightObj.size() - 1);
-      return JoinUtil.isFiltered(filter.get(), left);
-    }
-    return false;
-  }
-
-  // returns object has any filtered tag
-  private boolean hasAnyFiltered(int alias, List<Object> row) {
-    return row == dummyObj[alias] ||
-        hasFilter(alias) && JoinUtil.hasAnyFiltered(((ByteWritable) row.get(row.size() - 1)).get());
-  }
-
-  protected final boolean hasFilter(int alias) {
-    return filterMap != null && filterMap[alias] != null;
+    Arrays.fill(aliasFilterTags, (byte)0xff);
   }
 
   protected void reportProgress() {

Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java?rev=1462670&r1=1462669&r2=1462670&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java (original)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java Fri Mar 29 22:58:43 2013
@@ -45,7 +45,6 @@ import java.util.Set;
 import java.util.SortedSet;
 import java.util.TreeSet;
 
-import org.antlr.stringtemplate.StringTemplate;
 import org.apache.commons.lang.StringEscapeUtils;
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
@@ -165,6 +164,7 @@ import org.apache.hadoop.hive.shims.Hado
 import org.apache.hadoop.hive.shims.ShimLoader;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.util.ToolRunner;
+import org.stringtemplate.v4.ST;
 
 /**
  * DDLTask implementation.
@@ -1948,17 +1948,17 @@ public class DDLTask extends Task<DDLWor
         return 0;
       }
 
-      StringTemplate createTab_stmt = new StringTemplate("CREATE $" + EXTERNAL + "$ TABLE " +
+      ST createTab_stmt = new ST("CREATE <" + EXTERNAL + "> TABLE " +
           tableName + "(\n" +
-          "$" + LIST_COLUMNS + "$)\n" +
-          "$" + TBL_COMMENT + "$\n" +
-          "$" + LIST_PARTITIONS + "$\n" +
-          "$" + SORT_BUCKET + "$\n" +
-          "$" + ROW_FORMAT + "$\n" +
+          "<" + LIST_COLUMNS + ">)\n" +
+          "<" + TBL_COMMENT + ">\n" +
+          "<" + LIST_PARTITIONS + ">\n" +
+          "<" + SORT_BUCKET + ">\n" +
+          "<" + ROW_FORMAT + ">\n" +
           "LOCATION\n" +
-          "$" + TBL_LOCATION + "$\n" +
+          "<" + TBL_LOCATION + ">\n" +
           "TBLPROPERTIES (\n" +
-          "$" + TBL_PROPERTIES + "$)\n");
+          "<" + TBL_PROPERTIES + ">)\n");
 
       // For cases where the table is external
       String tbl_external = "";
@@ -2115,16 +2115,16 @@ public class DDLTask extends Task<DDLWor
         tbl_properties += StringUtils.join(realProps, ", \n");
       }
 
-      createTab_stmt.setAttribute(EXTERNAL, tbl_external);
-      createTab_stmt.setAttribute(LIST_COLUMNS, tbl_columns);
-      createTab_stmt.setAttribute(TBL_COMMENT, tbl_comment);
-      createTab_stmt.setAttribute(LIST_PARTITIONS, tbl_partitions);
-      createTab_stmt.setAttribute(SORT_BUCKET, tbl_sort_bucket);
-      createTab_stmt.setAttribute(ROW_FORMAT, tbl_row_format);
-      createTab_stmt.setAttribute(TBL_LOCATION, tbl_location);
-      createTab_stmt.setAttribute(TBL_PROPERTIES, tbl_properties);
+      createTab_stmt.add(EXTERNAL, tbl_external);
+      createTab_stmt.add(LIST_COLUMNS, tbl_columns);
+      createTab_stmt.add(TBL_COMMENT, tbl_comment);
+      createTab_stmt.add(LIST_PARTITIONS, tbl_partitions);
+      createTab_stmt.add(SORT_BUCKET, tbl_sort_bucket);
+      createTab_stmt.add(ROW_FORMAT, tbl_row_format);
+      createTab_stmt.add(TBL_LOCATION, tbl_location);
+      createTab_stmt.add(TBL_PROPERTIES, tbl_properties);
 
-      outStream.writeBytes(createTab_stmt.toString());
+      outStream.writeBytes(createTab_stmt.render());
       ((FSDataOutputStream) outStream).close();
       outStream = null;
     } catch (FileNotFoundException e) {

Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java?rev=1462670&r1=1462669&r2=1462670&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java (original)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java Fri Mar 29 22:58:43 2013
@@ -81,7 +81,7 @@ public class HashTableSinkOperator exten
    */
   protected transient List<ExprNodeEvaluator>[] joinFilters;
 
-  protected transient int[][] filterMap;
+  protected transient int[][] filterMaps;
 
   protected transient int numAliases; // number of aliases
   /**
@@ -121,16 +121,18 @@ public class HashTableSinkOperator exten
     SerDe serde;
     TableDesc tblDesc;
     Configuration conf;
+    boolean hasFilter;
 
     /**
      * @param standardOI
      * @param serde
      */
     public HashTableSinkObjectCtx(ObjectInspector standardOI, SerDe serde, TableDesc tblDesc,
-        Configuration conf) {
+        boolean hasFilter, Configuration conf) {
       this.standardOI = standardOI;
       this.serde = serde;
       this.tblDesc = tblDesc;
+      this.hasFilter = hasFilter;
       this.conf = conf;
     }
 
@@ -152,6 +154,10 @@ public class HashTableSinkOperator exten
       return tblDesc;
     }
 
+    public boolean hasFilterTag() {
+      return hasFilter;
+    }
+
     public Configuration getConf() {
       return conf;
     }
@@ -193,7 +199,7 @@ public class HashTableSinkOperator exten
     totalSz = 0;
 
     noOuterJoin = conf.isNoOuterJoin();
-    filterMap = conf.getFilterMap();
+    filterMaps = conf.getFilterMap();
 
     int tagLen = conf.getTagLength();
 
@@ -228,10 +234,10 @@ public class HashTableSinkOperator exten
           continue;
         }
         List<ObjectInspector> rcOIs = joinValuesObjectInspectors[alias];
-        if (filterMap != null && filterMap[alias] != null) {
+        if (filterMaps != null && filterMaps[alias] != null) {
           // for each alias, add object inspector for filter tag as the last element
           rcOIs = new ArrayList<ObjectInspector>(rcOIs);
-          rcOIs.add(PrimitiveObjectInspectorFactory.writableByteObjectInspector);
+          rcOIs.add(PrimitiveObjectInspectorFactory.writableShortObjectInspector);
         }
         rowContainerObjectInspectors[alias] = rcOIs;
       }
@@ -298,9 +304,12 @@ public class HashTableSinkOperator exten
     MapJoinMetaData.clear();
     MapJoinMetaData.put(Integer.valueOf(metadataKeyTag), new HashTableSinkObjectCtx(
         ObjectInspectorUtils.getStandardObjectInspector(keySerializer.getObjectInspector(),
-            ObjectInspectorCopyOption.WRITABLE), keySerializer, keyTableDesc, hconf));
+            ObjectInspectorCopyOption.WRITABLE), keySerializer, keyTableDesc, false, hconf));
   }
 
+  private boolean hasFilter(int alias) {
+    return filterMaps != null && filterMaps[alias] != null;
+  }
   /*
    * This operator only process small tables Read the key/value pairs Load them into hashtable
    */
@@ -320,9 +329,8 @@ public class HashTableSinkOperator exten
           joinKeysObjectInspectors[alias]);
 
       Object[] value = JoinUtil.computeMapJoinValues(row, joinValues[alias],
-          joinValuesObjectInspectors[alias], joinFilters[alias], joinFilterObjectInspectors
-              [alias], filterMap == null ? null : filterMap[alias]);
-
+          joinValuesObjectInspectors[alias], joinFilters[alias], joinFilterObjectInspectors[alias],
+          filterMaps == null ? null : filterMaps[alias]);
 
       HashMapWrapper<AbstractMapJoinKey, MapJoinObjectValue> hashTable = mapJoinTables[alias];
 
@@ -382,8 +390,9 @@ public class HashTableSinkOperator exten
     StandardStructObjectInspector standardOI = ObjectInspectorFactory
         .getStandardStructObjectInspector(newNames, newFields);
 
-    MapJoinMetaData.put(Integer.valueOf(metadataValueTag[tag]), new HashTableSinkObjectCtx(
-        standardOI, valueSerDe, valueTableDesc, hconf));
+    int alias = Integer.valueOf(metadataValueTag[tag]);
+    MapJoinMetaData.put(alias, new HashTableSinkObjectCtx(
+        standardOI, valueSerDe, valueTableDesc, hasFilter(alias), hconf));
   }
 
   @Override

Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java?rev=1462670&r1=1462669&r2=1462670&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java (original)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java Fri Mar 29 22:58:43 2013
@@ -80,12 +80,7 @@ public class JoinOperator extends Common
         nextSz = joinEmitInterval;
       }
 
-
-      ArrayList<Object> nr = JoinUtil.computeValues(row, joinValues[alias],
-          joinValuesObjectInspectors[alias], joinFilters[alias],
-          joinFilterObjectInspectors[alias],
-          filterMap == null ? null : filterMap[alias]);
-
+      ArrayList<Object> nr = getFilteredValue(alias, row);
 
       if (handleSkewJoin) {
         skewJoinKeyContext.handleSkew(tag);

Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinUtil.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinUtil.java?rev=1462670&r1=1462669&r2=1462670&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinUtil.java (original)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinUtil.java Fri Mar 29 22:58:43 2013
@@ -35,15 +35,15 @@ import org.apache.hadoop.hive.ql.plan.Jo
 import org.apache.hadoop.hive.ql.plan.TableDesc;
 import org.apache.hadoop.hive.serde2.SerDe;
 import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.serde2.io.ShortWritable;
 import org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption;
 import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption;
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
-import org.apache.hadoop.hive.serde2.io.ByteWritable;
 import org.apache.hadoop.mapred.Reporter;
 import org.apache.hadoop.mapred.SequenceFileInputFormat;
 import org.apache.hadoop.util.ReflectionUtils;
@@ -203,7 +203,7 @@ public class JoinUtil {
     if (filterMap != null) {
       nr = new Object[valueFields.size()+1];
       // add whether the row is filtered or not.
-      nr[valueFields.size()] = new ByteWritable(isFiltered(row, filters, filtersOI, filterMap));
+      nr[valueFields.size()] = new ShortWritable(isFiltered(row, filters, filtersOI, filterMap));
     }else{
       nr = new Object[valueFields.size()];
     }
@@ -220,37 +220,40 @@ public class JoinUtil {
   /**
    * Return the value as a standard object. StandardObject can be inspected by a
    * standard ObjectInspector.
+   * If it would be tagged by filter, reserve one more slot for that.
    */
   public static ArrayList<Object> computeValues(Object row,
-      List<ExprNodeEvaluator> valueFields, List<ObjectInspector> valueFieldsOI,
-      List<ExprNodeEvaluator> filters, List<ObjectInspector> filtersOI,
-      int[] filterMap) throws HiveException {
+      List<ExprNodeEvaluator> valueFields, List<ObjectInspector> valueFieldsOI, boolean hasFilter)
+      throws HiveException {
 
     // Compute the values
-    ArrayList<Object> nr = new ArrayList<Object>(valueFields.size());
+    int reserve = hasFilter ? valueFields.size() + 1 : valueFields.size();
+    ArrayList<Object> nr = new ArrayList<Object>(reserve);
     for (int i = 0; i < valueFields.size(); i++) {
       nr.add(ObjectInspectorUtils.copyToStandardObject(valueFields.get(i)
           .evaluate(row), valueFieldsOI.get(i),
           ObjectInspectorCopyOption.WRITABLE));
     }
-    if (filterMap != null) {
-      // add whether the row is filtered or not.
-      nr.add(new ByteWritable(isFiltered(row, filters, filtersOI, filterMap)));
-    }
-
     return nr;
   }
 
-  private static final byte[] MASKS = new byte[]
-      {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, (byte) 0x80};
+  private static final short[] MASKS;
+  static {
+    int num = 32;
+    MASKS = new short[num];
+    MASKS[0] = 1;
+    for (int idx = 1; idx < num; idx++) {
+      MASKS[idx] = (short)(2 * MASKS[idx-1]);
+    }
+  }
 
   /**
    * Returns true if the row does not pass through filters.
    */
-  protected static byte isFiltered(Object row, List<ExprNodeEvaluator> filters,
+  protected static short isFiltered(Object row, List<ExprNodeEvaluator> filters,
       List<ObjectInspector> ois, int[] filterMap) throws HiveException {
     // apply join filters on the row.
-    byte ret = 0;
+    short ret = 0;
     int j = 0;
     for (int i = 0; i < filterMap.length; i += 2) {
       int tag = filterMap[i];
@@ -274,11 +277,11 @@ public class JoinUtil {
     return ret;
   }
 
-  protected static boolean isFiltered(byte filter, int tag) {
+  protected static boolean isFiltered(short filter, int tag) {
     return (filter & MASKS[tag]) != 0;
   }
 
-  protected static boolean hasAnyFiltered(byte tag) {
+  protected static boolean hasAnyFiltered(short tag) {
     return tag != 0;
   }
 
@@ -330,7 +333,7 @@ public class JoinUtil {
       if (!noFilter) {
         colNames.append("filtered");
         colNames.append(',');
-        colTypes.append(TypeInfoFactory.byteTypeInfo.getTypeName());
+        colTypes.append(TypeInfoFactory.shortTypeInfo.getTypeName());
         colTypes.append(',');
       }
       // remove the last ','

Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java?rev=1462670&r1=1462669&r2=1462670&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java (original)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java Fri Mar 29 22:58:43 2013
@@ -38,6 +38,7 @@ import org.apache.hadoop.hive.ql.plan.Ta
 import org.apache.hadoop.hive.ql.plan.api.OperatorType;
 import org.apache.hadoop.hive.serde2.SerDe;
 import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption;
 import org.apache.hadoop.hive.shims.ShimLoader;
@@ -118,7 +119,7 @@ public class MapJoinOperator extends Abs
     keySerializer.initialize(null, keyTableDesc.getProperties());
     MapJoinMetaData.put(Integer.valueOf(metadataKeyTag), new HashTableSinkObjectCtx(
         ObjectInspectorUtils.getStandardObjectInspector(keySerializer.getObjectInspector(),
-            ObjectInspectorCopyOption.WRITABLE), keySerializer, keyTableDesc, hconf));
+            ObjectInspectorCopyOption.WRITABLE), keySerializer, keyTableDesc, false, hconf));
 
     for (int pos = 0; pos < order.length; pos++) {
       if (pos == posBigTable) {
@@ -134,9 +135,10 @@ public class MapJoinOperator extends Abs
           null);
       valueSerDe.initialize(null, valueTableDesc.getProperties());
 
+      ObjectInspector inspector = valueSerDe.getObjectInspector();
       MapJoinMetaData.put(Integer.valueOf(pos), new HashTableSinkObjectCtx(ObjectInspectorUtils
-          .getStandardObjectInspector(valueSerDe.getObjectInspector(),
-              ObjectInspectorCopyOption.WRITABLE), valueSerDe, valueTableDesc, hconf));
+          .getStandardObjectInspector(inspector, ObjectInspectorCopyOption.WRITABLE),
+          valueSerDe, valueTableDesc, hasFilter(pos), hconf));
     }
   }
 
@@ -228,10 +230,7 @@ public class MapJoinOperator extends Abs
       // compute keys and values as StandardObjects
       AbstractMapJoinKey key = JoinUtil.computeMapJoinKeys(row, joinKeys[alias],
           joinKeysObjectInspectors[alias]);
-      ArrayList<Object> value = JoinUtil.computeValues(row, joinValues[alias],
-          joinValuesObjectInspectors[alias], joinFilters[alias], joinFilterObjectInspectors
-              [alias], filterMap == null ? null : filterMap[alias]);
-
+      ArrayList<Object> value = getFilteredValue(alias, row);
 
       // Add the value to the ArrayList
       storage[alias].add(value);
@@ -252,6 +251,7 @@ public class MapJoinOperator extends Abs
           } else {
             rowContainer.reset(o.getObj());
             storage[pos] = rowContainer;
+            aliasFilterTags[pos] = o.getAliasFilter();
           }
         }
       }

Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java?rev=1462670&r1=1462669&r2=1462670&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java (original)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java Fri Mar 29 22:58:43 2013
@@ -256,10 +256,7 @@ public class SMBMapJoinOperator extends 
     // compute keys and values as StandardObjects
     ArrayList<Object> key = JoinUtil.computeKeys(row, joinKeys[alias],
         joinKeysObjectInspectors[alias]);
-    ArrayList<Object> value = JoinUtil.computeValues(row, joinValues[alias],
-        joinValuesObjectInspectors[alias], joinFilters[alias],
-        joinFilterObjectInspectors[alias],
-        filterMap == null ? null : filterMap[alias]);
+    ArrayList<Object> value = getFilteredValue(alias, row);
 
 
     //have we reached a new key group?

Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java?rev=1462670&r1=1462669&r2=1462670&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java (original)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java Fri Mar 29 22:58:43 2013
@@ -392,6 +392,7 @@ public class StatsTask extends Task<Stat
           if (statsAggregator != null) {
             updateStats(collectableStats, newPartStats, statsAggregator,
                 parameters, partitionID, atomic);
+            statsAggregator.cleanUp(partitionID);
           } else {
             for (String statType : collectableStats) {
               // The collectable stats for the aggregator needs to be cleared.
@@ -441,10 +442,6 @@ public class StatsTask extends Task<Stat
           String tableFullName = table.getDbName() + "." + table.getTableName();
           db.alterPartition(tableFullName, new Partition(table, tPart));
 
-          if (statsAggregator != null) {
-            statsAggregator.cleanUp(partitionID);
-          }
-
           console.printInfo("Partition " + tableFullName + partn.getSpec() +
               " stats: [" + newPartStats.toString() + ']');
         }

Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java?rev=1462670&r1=1462669&r2=1462670&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java (original)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java Fri Mar 29 22:58:43 2013
@@ -25,6 +25,8 @@ import java.util.List;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.io.rcfile.merge.BlockMergeTask;
 import org.apache.hadoop.hive.ql.io.rcfile.merge.MergeWork;
+import org.apache.hadoop.hive.ql.io.rcfile.stats.PartialScanTask;
+import org.apache.hadoop.hive.ql.io.rcfile.stats.PartialScanWork;
 import org.apache.hadoop.hive.ql.plan.ColumnStatsWork;
 import org.apache.hadoop.hive.ql.plan.ConditionalWork;
 import org.apache.hadoop.hive.ql.plan.CopyWork;
@@ -83,6 +85,8 @@ public final class TaskFactory {
         BlockMergeTask.class));
     taskvec.add(new taskTuple<DependencyCollectionWork>(DependencyCollectionWork.class,
         DependencyCollectionTask.class));
+    taskvec.add(new taskTuple<PartialScanWork>(PartialScanWork.class,
+        PartialScanTask.class));
 
   }
 

Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinObjectValue.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinObjectValue.java?rev=1462670&r1=1462669&r2=1462670&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinObjectValue.java (original)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinObjectValue.java Fri Mar 29 22:58:43 2013
@@ -28,6 +28,7 @@ import org.apache.hadoop.hive.ql.exec.Ma
 import org.apache.hadoop.hive.ql.exec.HashTableSinkOperator.HashTableSinkObjectCtx;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.serde2.io.ShortWritable;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption;
 import org.apache.hadoop.io.Writable;
@@ -40,7 +41,7 @@ public class MapJoinObjectValue implemen
   protected transient int metadataTag;
   protected transient MapJoinRowContainer<Object[]> obj;
 
-
+  protected transient byte aliasFilter = (byte) 0xff;
 
   public MapJoinObjectValue() {
 
@@ -55,6 +56,10 @@ public class MapJoinObjectValue implemen
     this.obj = obj;
   }
 
+  public byte getAliasFilter() {
+    return aliasFilter;
+  }
+
   @Override
   public boolean equals(Object o) {
     if (o instanceof MapJoinObjectValue) {
@@ -86,7 +91,6 @@ public class MapJoinObjectValue implemen
       // get the tableDesc from the map stored in the mapjoin operator
       HashTableSinkObjectCtx ctx = MapJoinMetaData.get(Integer.valueOf(metadataTag));
       int sz = in.readInt();
-
       MapJoinRowContainer<Object[]> res = new MapJoinRowContainer<Object[]>();
       if (sz > 0) {
         int numCols = in.readInt();
@@ -102,7 +106,11 @@ public class MapJoinObjectValue implemen
             if (memObj == null) {
               res.add(new ArrayList<Object>(0).toArray());
             } else {
-              res.add(memObj.toArray());
+              Object[] array = memObj.toArray();
+              res.add(array);
+              if (ctx.hasFilterTag()) {
+                aliasFilter &= ((ShortWritable)array[array.length - 1]).get();
+              }
             }
           }
         } else {

Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java?rev=1462670&r1=1462669&r2=1462670&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java (original)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java Fri Mar 29 22:58:43 2013
@@ -17,6 +17,15 @@
  */
 package org.apache.hadoop.hive.ql.io.orc;
 
+import java.io.EOFException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.sql.Timestamp;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -30,15 +39,6 @@ import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.Text;
 
-import java.io.EOFException;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.sql.Timestamp;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
 class RecordReaderImpl implements RecordReader {
   private final FSDataInputStream file;
   private final long firstRow;
@@ -686,7 +686,13 @@ class RecordReaderImpl implements Record
         } else {
           length = dictionaryBuffer.size() - offset;
         }
-        dictionaryBuffer.setText(result, offset, length);
+        // If the column is just empty strings, the size will be zero, so the buffer will be null,
+        // in that case just return result as it will default to empty
+        if (dictionaryBuffer != null) {
+          dictionaryBuffer.setText(result, offset, length);
+        } else {
+          result.clear();
+        }
       }
       return result;
     }

Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/RCFileKeyBufferWrapper.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/RCFileKeyBufferWrapper.java?rev=1462670&r1=1462669&r2=1462670&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/RCFileKeyBufferWrapper.java (original)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/RCFileKeyBufferWrapper.java Fri Mar 29 22:58:43 2013
@@ -62,4 +62,8 @@ public class RCFileKeyBufferWrapper impl
     return this.keyBuffer.compareTo(o.keyBuffer);
   }
 
+  public KeyBuffer getKeyBuffer() {
+    return keyBuffer;
+  }
+
 }



Mime
View raw message