Return-Path: X-Original-To: apmail-hive-commits-archive@www.apache.org Delivered-To: apmail-hive-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 9E79DF5E9 for ; Fri, 4 Oct 2013 23:36:42 +0000 (UTC) Received: (qmail 91189 invoked by uid 500); 4 Oct 2013 23:36:42 -0000 Delivered-To: apmail-hive-commits-archive@hive.apache.org Received: (qmail 91074 invoked by uid 500); 4 Oct 2013 23:36:42 -0000 Mailing-List: contact commits-help@hive.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hive-dev@hive.apache.org Delivered-To: mailing list commits@hive.apache.org Received: (qmail 90957 invoked by uid 99); 4 Oct 2013 23:36:42 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 04 Oct 2013 23:36:42 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 04 Oct 2013 23:36:21 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id D1F0F2388B34; Fri, 4 Oct 2013 23:35:58 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1529347 [4/6] - in /hive/branches/tez: ./ ant/src/org/apache/hadoop/hive/ant/ beeline/src/java/org/apache/hive/beeline/ beeline/src/test/org/apache/hive/beeline/src/test/ bin/ common/ common/src/test/org/apache/hadoop/hive/common/type/ dat... Date: Fri, 04 Oct 2013 23:35:53 -0000 To: commits@hive.apache.org From: gunther@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20131004233558.D1F0F2388B34@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionEvaluator.java URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionEvaluator.java?rev=1529347&r1=1529346&r2=1529347&view=diff ============================================================================== --- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionEvaluator.java (original) +++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionEvaluator.java Fri Oct 4 23:35:50 2013 @@ -24,7 +24,7 @@ import org.apache.hadoop.hive.ql.exec.PT import org.apache.hadoop.hive.ql.exec.PTFUtils; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.PTFDesc; -import org.apache.hadoop.hive.ql.plan.PTFDesc.PartitionedTableFunctionDef; +import org.apache.hadoop.hive.ql.plan.ptf.PartitionedTableFunctionDef; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; @@ -46,8 +46,7 @@ import org.apache.hadoop.hive.serde2.obj * * */ -public abstract class TableFunctionEvaluator -{ +public abstract class TableFunctionEvaluator { /* * how is this different from the OutpuShape set on the TableDef. * This is the OI of the object coming out of the PTF. @@ -59,52 +58,45 @@ public abstract class TableFunctionEvalu * same comment as OI applies here. */ transient protected StructObjectInspector rawInputOI; - protected PartitionedTableFunctionDef tDef; + protected PartitionedTableFunctionDef tableDef; protected PTFDesc ptfDesc; boolean transformsRawInput; transient protected PTFPartition outputPartition; - static{ + static { + //TODO is this a bug? The field is not named outputOI it is named OI PTFUtils.makeTransient(TableFunctionEvaluator.class, "outputOI", "rawInputOI"); } - public StructObjectInspector getOutputOI() - { + public StructObjectInspector getOutputOI() { return OI; } - protected void setOutputOI(StructObjectInspector outputOI) - { + protected void setOutputOI(StructObjectInspector outputOI) { OI = outputOI; } - public PartitionedTableFunctionDef getTableDef() - { - return tDef; + public PartitionedTableFunctionDef getTableDef() { + return tableDef; } - public void setTableDef(PartitionedTableFunctionDef tDef) - { - this.tDef = tDef; + public void setTableDef(PartitionedTableFunctionDef tDef) { + this.tableDef = tDef; } - protected PTFDesc getQueryDef() - { + protected PTFDesc getQueryDef() { return ptfDesc; } - protected void setQueryDef(PTFDesc ptfDesc) - { + protected void setQueryDef(PTFDesc ptfDesc) { this.ptfDesc = ptfDesc; } - public StructObjectInspector getRawInputOI() - { + public StructObjectInspector getRawInputOI() { return rawInputOI; } - protected void setRawInputOI(StructObjectInspector rawInputOI) - { + protected void setRawInputOI(StructObjectInspector rawInputOI) { this.rawInputOI = rawInputOI; } @@ -117,17 +109,15 @@ public abstract class TableFunctionEvalu } public PTFPartition execute(PTFPartition iPart) - throws HiveException - { + throws HiveException { PTFPartitionIterator pItr = iPart.iterator(); PTFOperator.connectLeadLagFunctionsToPartition(ptfDesc, pItr); if ( outputPartition == null ) { outputPartition = PTFPartition.create(ptfDesc.getCfg(), - tDef.getOutputShape().getSerde(), - OI, tDef.getOutputShape().getOI()); - } - else { + tableDef.getOutputShape().getSerde(), + OI, tableDef.getOutputShape().getOI()); + } else { outputPartition.reset(); } @@ -137,17 +127,14 @@ public abstract class TableFunctionEvalu protected abstract void execute(PTFPartitionIterator pItr, PTFPartition oPart) throws HiveException; - public PTFPartition transformRawInput(PTFPartition iPart) throws HiveException - { - if ( !isTransformsRawInput()) - { - throw new HiveException(String.format("Internal Error: mapExecute called on function (%s)that has no Map Phase", tDef.getName())); + public PTFPartition transformRawInput(PTFPartition iPart) throws HiveException { + if (!isTransformsRawInput()) { + throw new HiveException(String.format("Internal Error: mapExecute called on function (%s)that has no Map Phase", tableDef.getName())); } return _transformRawInput(iPart); } - protected PTFPartition _transformRawInput(PTFPartition iPart) throws HiveException - { + protected PTFPartition _transformRawInput(PTFPartition iPart) throws HiveException { return null; } } Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionResolver.java URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionResolver.java?rev=1529347&r1=1529346&r2=1529347&view=diff ============================================================================== --- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionResolver.java (original) +++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionResolver.java Fri Oct 4 23:35:50 2013 @@ -18,7 +18,7 @@ package org.apache.hadoop.hive.ql.udf.ptf; -import java.util.ArrayList; +import java.util.List; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator; @@ -27,7 +27,7 @@ import org.apache.hadoop.hive.ql.metadat import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.PTFDesc; -import org.apache.hadoop.hive.ql.plan.PTFDesc.PartitionedTableFunctionDef; +import org.apache.hadoop.hive.ql.plan.ptf.PartitionedTableFunctionDef; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFResolver; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; @@ -53,8 +53,7 @@ import org.apache.hadoop.hive.serde2.obj * */ @SuppressWarnings("deprecation") -public abstract class TableFunctionResolver -{ +public abstract class TableFunctionResolver { TableFunctionEvaluator evaluator; PTFDesc ptfDesc; @@ -65,8 +64,7 @@ public abstract class TableFunctionResol * the transformsRawInput boolean. */ public void initialize(HiveConf cfg, PTFDesc ptfDesc, PartitionedTableFunctionDef tDef) - throws SemanticException - { + throws SemanticException { this.ptfDesc = ptfDesc; evaluator = createEvaluator(ptfDesc, tDef); @@ -79,16 +77,14 @@ public abstract class TableFunctionResol * called during deserialization of a QueryDef during runtime. */ public void initialize(PTFDesc ptfDesc, PartitionedTableFunctionDef tDef, TableFunctionEvaluator evaluator) - throws HiveException - { + throws HiveException { this.evaluator = evaluator; this.ptfDesc = ptfDesc; evaluator.setTableDef(tDef); evaluator.setQueryDef(ptfDesc); } - public TableFunctionEvaluator getEvaluator() - { + public TableFunctionEvaluator getEvaluator() { return evaluator; } @@ -105,7 +101,7 @@ public abstract class TableFunctionResol * A PTF Function must provide the 'external' names of the columns in its Output. * */ - public abstract ArrayList getOutputColumnNames() throws SemanticException; + public abstract List getOutputColumnNames() throws SemanticException; /** @@ -127,10 +123,8 @@ public abstract class TableFunctionResol * - subsequent to this call, a call to getRawInputOI call on the {@link TableFunctionEvaluator} must return the OI * of the output of this function. */ - public void setupRawInputOI() throws SemanticException - { - if (!transformsRawInput()) - { + public void setupRawInputOI() throws SemanticException { + if (!transformsRawInput()) { return; } throw new SemanticException( @@ -141,9 +135,8 @@ public abstract class TableFunctionResol * A PTF Function must provide the 'external' names of the columns in the transformed Raw Input. * */ - public ArrayList getRawInputColumnNames() throws SemanticException { - if (!transformsRawInput()) - { + public List getRawInputColumnNames() throws SemanticException { + if (!transformsRawInput()) { return null; } throw new SemanticException( @@ -153,10 +146,8 @@ public abstract class TableFunctionResol /* * Same responsibility as initializeOI, but for the RawInput. */ - public void initializeRawInputOI() throws HiveException - { - if (!transformsRawInput()) - { + public void initializeRawInputOI() throws HiveException { + if (!transformsRawInput()) { return; } throw new HiveException( @@ -166,21 +157,18 @@ public abstract class TableFunctionResol /* * callback method used by subclasses to set the RawInputOI on the Evaluator. */ - protected void setRawInputOI(StructObjectInspector rawInputOI) - { + protected void setRawInputOI(StructObjectInspector rawInputOI) { evaluator.setRawInputOI(rawInputOI); } /* * callback method used by subclasses to set the OutputOI on the Evaluator. */ - protected void setOutputOI(StructObjectInspector outputOI) - { + protected void setOutputOI(StructObjectInspector outputOI) { evaluator.setOutputOI(outputOI); } - public PTFDesc getPtfDesc() - { + public PTFDesc getPtfDesc() { return ptfDesc; } Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/WindowingTableFunction.java URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/WindowingTableFunction.java?rev=1529347&r1=1529346&r2=1529347&view=diff ============================================================================== --- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/WindowingTableFunction.java (original) +++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/WindowingTableFunction.java Fri Oct 4 23:35:50 2013 @@ -31,13 +31,13 @@ import org.apache.hadoop.hive.ql.parse.S import org.apache.hadoop.hive.ql.parse.WindowingSpec.BoundarySpec; import org.apache.hadoop.hive.ql.parse.WindowingSpec.Direction; import org.apache.hadoop.hive.ql.plan.PTFDesc; -import org.apache.hadoop.hive.ql.plan.PTFDesc.BoundaryDef; -import org.apache.hadoop.hive.ql.plan.PTFDesc.PTFExpressionDef; -import org.apache.hadoop.hive.ql.plan.PTFDesc.PartitionedTableFunctionDef; -import org.apache.hadoop.hive.ql.plan.PTFDesc.ValueBoundaryDef; -import org.apache.hadoop.hive.ql.plan.PTFDesc.WindowFrameDef; -import org.apache.hadoop.hive.ql.plan.PTFDesc.WindowFunctionDef; -import org.apache.hadoop.hive.ql.plan.PTFDesc.WindowTableFunctionDef; +import org.apache.hadoop.hive.ql.plan.ptf.BoundaryDef; +import org.apache.hadoop.hive.ql.plan.ptf.PTFExpressionDef; +import org.apache.hadoop.hive.ql.plan.ptf.PartitionedTableFunctionDef; +import org.apache.hadoop.hive.ql.plan.ptf.ValueBoundaryDef; +import org.apache.hadoop.hive.ql.plan.ptf.WindowFrameDef; +import org.apache.hadoop.hive.ql.plan.ptf.WindowFunctionDef; +import org.apache.hadoop.hive.ql.plan.ptf.WindowTableFunctionDef; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.AggregationBuffer; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/util/JavaDataModel.java URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/util/JavaDataModel.java?rev=1529347&r1=1529346&r2=1529347&view=diff ============================================================================== --- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/util/JavaDataModel.java (original) +++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/util/JavaDataModel.java Fri Oct 4 23:35:50 2013 @@ -157,7 +157,7 @@ public enum JavaDataModel { // ascii string public int lengthFor(String string) { - return object() + primitive1() * 3 + array() + string.length(); + return lengthForStringOfLength(string.length()); } public int lengthFor(NumericHistogram histogram) { @@ -267,4 +267,33 @@ public enum JavaDataModel { public int lengthForBooleanArrayOfSize(int length) { return lengthForPrimitiveArrayOfSize(PRIMITIVE_BYTE, length); } + + public int lengthOfDecimal() { + // object overhead + 8 bytes for intCompact + 4 bytes for precision + // + 4 bytes for scale + size of BigInteger + return object() + 2 * primitive2() + lengthOfBigInteger(); + } + + private int lengthOfBigInteger() { + // object overhead + 4 bytes for bitCount + 4 bytes for bitLength + // + 4 bytes for firstNonzeroByteNum + 4 bytes for firstNonzeroIntNum + + // + 4 bytes for lowestSetBit + 5 bytes for size of magnitude (since max precision + // is only 38 for HiveDecimal) + 7 bytes of padding (since java memory allocations + // are 8 byte aligned) + return object() + 4 * primitive2(); + } + + public int lengthOfTimestamp() { + // object overhead + 4 bytes for int (nanos) + 4 bytes of padding + return object() + primitive2(); + } + + public int lengthOfDate() { + // object overhead + 8 bytes for long (fastTime) + 16 bytes for cdate + return object() + 3 * primitive2(); + } + + public int lengthForStringOfLength(int strLen) { + return object() + primitive1() * 3 + array() + strLen; + } } Modified: hive/branches/tez/ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto?rev=1529347&r1=1529346&r2=1529347&view=diff ============================================================================== --- hive/branches/tez/ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto (original) +++ hive/branches/tez/ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto Fri Oct 4 23:35:50 2013 @@ -15,6 +15,8 @@ message DoubleStatistics { message StringStatistics { optional string minimum = 1; optional string maximum = 2; + // sum will store the total length of all strings in a stripe + optional sint64 sum = 3; } message BucketStatistics { @@ -33,6 +35,11 @@ message DateStatistics { optional sint32 maximum = 2; } +message BinaryStatistics { + // sum will store the total binary blob length in a stripe + optional sint64 sum = 1; +} + message ColumnStatistics { optional uint64 numberOfValues = 1; optional IntegerStatistics intStatistics = 2; @@ -41,6 +48,7 @@ message ColumnStatistics { optional BucketStatistics bucketStatistics = 5; optional DecimalStatistics decimalStatistics = 6; optional DateStatistics dateStatistics = 7; + optional BinaryStatistics binaryStatistics = 8; } message RowIndexEntry { Modified: hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java?rev=1529347&r1=1529346&r2=1529347&view=diff ============================================================================== --- hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java (original) +++ hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java Fri Oct 4 23:35:50 2013 @@ -315,12 +315,7 @@ public class QTestUtil { testFiles = dataDir; - String ow = System.getProperty("test.output.overwrite"); - if ((ow != null) && ow.equalsIgnoreCase("true")) { - overWrite = true; - } else { - overWrite = false; - } + overWrite = "true".equalsIgnoreCase(System.getProperty("test.output.overwrite")); setup = new QTestSetup(); setup.preTest(conf); @@ -1534,4 +1529,15 @@ public class QTestUtil { + "or try \"ant test ... -Dtest.silent=false\" to get more logs."); System.err.flush(); } + + public static String ensurePathEndsInSlash(String path) { + if(path == null) { + throw new NullPointerException("Path cannot be null"); + } + if(path.endsWith(File.separator)) { + return path; + } else { + return path + File.separator; + } + } } Modified: hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorStringExpressions.java URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorStringExpressions.java?rev=1529347&r1=1529346&r2=1529347&view=diff ============================================================================== --- hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorStringExpressions.java (original) +++ hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorStringExpressions.java Fri Oct 4 23:35:50 2013 @@ -670,28 +670,28 @@ public class TestVectorStringExpressions // BEGIN pattern expr = new FilterStringColLikeStringScalar(0, new Text("abc%")); - Assert.assertEquals(FilterStringColLikeStringScalar.PatternType.BEGIN, - expr.getType()); + Assert.assertEquals(FilterStringColLikeStringScalar.BeginChecker.class, + expr.checker.getClass()); // END pattern expr = new FilterStringColLikeStringScalar(0, new Text("%abc")); - Assert.assertEquals(FilterStringColLikeStringScalar.PatternType.END, - expr.getType()); + Assert.assertEquals(FilterStringColLikeStringScalar.EndChecker.class, + expr.checker.getClass()); // MIDDLE pattern expr = new FilterStringColLikeStringScalar(0, new Text("%abc%")); - Assert.assertEquals(FilterStringColLikeStringScalar.PatternType.MIDDLE, - expr.getType()); + Assert.assertEquals(FilterStringColLikeStringScalar.MiddleChecker.class, + expr.checker.getClass()); // COMPLEX pattern expr = new FilterStringColLikeStringScalar(0, new Text("%abc%de")); - Assert.assertEquals(FilterStringColLikeStringScalar.PatternType.COMPLEX, - expr.getType()); + Assert.assertEquals(FilterStringColLikeStringScalar.ComplexChecker.class, + expr.checker.getClass()); // NONE pattern expr = new FilterStringColLikeStringScalar(0, new Text("abc")); - Assert.assertEquals(FilterStringColLikeStringScalar.PatternType.NONE, - expr.getType()); + Assert.assertEquals(FilterStringColLikeStringScalar.NoneChecker.class, + expr.checker.getClass()); } public void testStringLikeMultiByte() { Modified: hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java?rev=1529347&r1=1529346&r2=1529347&view=diff ============================================================================== --- hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java (original) +++ hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java Fri Oct 4 23:35:50 2013 @@ -18,12 +18,28 @@ package org.apache.hadoop.hive.ql.io.orc; +import static junit.framework.Assert.assertEquals; +import static junit.framework.Assert.assertNotNull; +import static junit.framework.Assert.assertNull; +import static junit.framework.Assert.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.math.BigInteger; +import java.nio.ByteBuffer; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Random; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.ql.io.sarg.SearchArgument; -import org.apache.hadoop.hive.ql.io.sarg.TestSearchArgumentImpl; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -55,26 +71,25 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TestName; -import java.io.File; -import java.io.IOException; -import java.math.BigInteger; -import java.nio.ByteBuffer; -import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Random; - -import static junit.framework.Assert.*; -import static junit.framework.Assert.assertEquals; - /** * Tests for the top level reader/streamFactory of ORC files. */ public class TestOrcFile { + public static class SimpleStruct { + BytesWritable bytes1; + Text string1; + + SimpleStruct(BytesWritable b1, String s1) { + this.bytes1 = b1; + if(s1 == null) { + this.string1 = null; + } else { + this.string1 = new Text(s1); + } + } + } + public static class InnerStruct { int int1; Text string1 = new Text(); @@ -132,48 +147,6 @@ public class TestOrcFile { } } - public static class AllTypesRow { - Boolean boolean1; - Byte byte1; - Short short1; - Integer int1; - Long long1; - Float float1; - Double double1; - BytesWritable bytes1; - Text string1; - MiddleStruct middle; - List list = new ArrayList(); - Map map = new HashMap(); - Timestamp ts; - HiveDecimal decimal1; - - AllTypesRow(Boolean b1, Byte b2, Short s1, Integer i1, Long l1, Float f1, - Double d1, - BytesWritable b3, String s2, MiddleStruct m1, - List l2, Map m2, - Timestamp ts1, HiveDecimal decimal) { - this.boolean1 = b1; - this.byte1 = b2; - this.short1 = s1; - this.int1 = i1; - this.long1 = l1; - this.float1 = f1; - this.double1 = d1; - this.bytes1 = b3; - if (s2 == null) { - this.string1 = null; - } else { - this.string1 = new Text(s2); - } - this.middle = m1; - this.list = l2; - this.map = m2; - this.ts = ts1; - this.decimal1 = decimal; - } - } - private static InnerStruct inner(int i, String s) { return new InnerStruct(i, s); } @@ -231,39 +204,6 @@ public class TestOrcFile { } @Test - public void testWriteFormat_0_11() throws Exception { - ObjectInspector inspector; - synchronized (TestOrcFile.class) { - inspector = ObjectInspectorFactory - .getReflectionObjectInspector(AllTypesRow.class, - ObjectInspectorFactory.ObjectInspectorOptions.JAVA); - } - conf.set("hive.exec.orc.write.format", "0.11"); - Writer writer = OrcFile.createWriter(fs, testFilePath, conf, inspector, - 100000, CompressionKind.NONE, 10000, 10000); - for(int i = 0; i < 7500; i++) { - if (i % 2 == 0) { - writer.addRow(new AllTypesRow(false, (byte) 1, (short) 1024, 65536, - Long.MAX_VALUE, (float) 1.0, -15.0, bytes(0, 1, 2, 3, 4), "hi", - new MiddleStruct(inner(1, "bye"), inner(2, "sigh")), list( - inner(3, "good"), inner(4, "bad")), map(), Timestamp - .valueOf("2000-03-12 15:00:00"), new HiveDecimal( - "12345678.6547456"))); - } else { - writer.addRow(new AllTypesRow(true, (byte) 100, (short) 2048, 65536, - Long.MAX_VALUE, (float) 2.0, -5.0, bytes(), "bye", - new MiddleStruct(inner(1, "bye"), inner(2, "sigh")), list( - inner(100000000, "cat"), inner(-100000, "in"), - inner(1234, "hat")), - map(inner(5, "chani"), inner(1, "mauddib")), Timestamp - .valueOf("2000-03-12 15:00:01"), new HiveDecimal( - "12345678.6547457"))); - } - } - writer.close(); - } - - @Test public void testReadFormat_0_11() throws Exception { Path resourceDir = new Path(System.getProperty("test.build.resources", "ql" + File.separator + "src" + File.separator + "test" + File.separator @@ -319,7 +259,7 @@ public class TestOrcFile { assertEquals("count: 7500 min: -15.0 max: -5.0 sum: -75000.0", stats[7].toString()); - assertEquals("count: 7500 min: bye max: hi", stats[9].toString()); + assertEquals("count: 7500 min: bye max: hi sum: 0", stats[9].toString()); // check the inspectors StructObjectInspector readerInspector = (StructObjectInspector) reader @@ -515,6 +455,93 @@ public class TestOrcFile { } @Test + public void testStringAndBinaryStatistics() throws Exception { + + ObjectInspector inspector; + synchronized (TestOrcFile.class) { + inspector = ObjectInspectorFactory.getReflectionObjectInspector + (SimpleStruct.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA); + } + Writer writer = OrcFile.createWriter(testFilePath, + OrcFile.writerOptions(conf) + .inspector(inspector) + .stripeSize(100000) + .bufferSize(10000)); + writer.addRow(new SimpleStruct(bytes(0,1,2,3,4), "foo")); + writer.addRow(new SimpleStruct(bytes(0,1,2,3), "bar")); + writer.addRow(new SimpleStruct(bytes(0,1,2,3,4,5), null)); + writer.addRow(new SimpleStruct(null, "hi")); + writer.close(); + Reader reader = OrcFile.createReader(fs, testFilePath); + + // check the stats + ColumnStatistics[] stats = reader.getStatistics(); + assertEquals(4, stats[0].getNumberOfValues()); + assertEquals("count: 4", stats[0].toString()); + + assertEquals(3, stats[1].getNumberOfValues()); + assertEquals(15, ((BinaryColumnStatistics) stats[1]).getSum()); + assertEquals("count: 3 sum: 15", stats[1].toString()); + + assertEquals(3, stats[2].getNumberOfValues()); + assertEquals("bar", ((StringColumnStatistics) stats[2]).getMinimum()); + assertEquals("hi", ((StringColumnStatistics) stats[2]).getMaximum()); + assertEquals(8, ((StringColumnStatistics) stats[2]).getSum()); + assertEquals("count: 3 min: bar max: hi sum: 8", + stats[2].toString()); + + // check the inspectors + StructObjectInspector readerInspector = + (StructObjectInspector) reader.getObjectInspector(); + assertEquals(ObjectInspector.Category.STRUCT, + readerInspector.getCategory()); + assertEquals("struct", + readerInspector.getTypeName()); + List fields = + readerInspector.getAllStructFieldRefs(); + BinaryObjectInspector bi = (BinaryObjectInspector) readerInspector. + getStructFieldRef("bytes1").getFieldObjectInspector(); + StringObjectInspector st = (StringObjectInspector) readerInspector. + getStructFieldRef("string1").getFieldObjectInspector(); + RecordReader rows = reader.rows(null); + Object row = rows.next(null); + assertNotNull(row); + // check the contents of the first row + assertEquals(bytes(0,1,2,3,4), bi.getPrimitiveWritableObject( + readerInspector.getStructFieldData(row, fields.get(0)))); + assertEquals("foo", st.getPrimitiveJavaObject(readerInspector. + getStructFieldData(row, fields.get(1)))); + + // check the contents of second row + assertEquals(true, rows.hasNext()); + row = rows.next(row); + assertEquals(bytes(0,1,2,3), bi.getPrimitiveWritableObject( + readerInspector.getStructFieldData(row, fields.get(0)))); + assertEquals("bar", st.getPrimitiveJavaObject(readerInspector. + getStructFieldData(row, fields.get(1)))); + + // check the contents of second row + assertEquals(true, rows.hasNext()); + row = rows.next(row); + assertEquals(bytes(0,1,2,3,4,5), bi.getPrimitiveWritableObject( + readerInspector.getStructFieldData(row, fields.get(0)))); + assertNull(st.getPrimitiveJavaObject(readerInspector. + getStructFieldData(row, fields.get(1)))); + + // check the contents of second row + assertEquals(true, rows.hasNext()); + row = rows.next(row); + assertNull(bi.getPrimitiveWritableObject( + readerInspector.getStructFieldData(row, fields.get(0)))); + assertEquals("hi", st.getPrimitiveJavaObject(readerInspector. + getStructFieldData(row, fields.get(1)))); + + // handle the close up + assertEquals(false, rows.hasNext()); + rows.close(); + } + + @Test public void test1() throws Exception { ObjectInspector inspector; synchronized (TestOrcFile.class) { @@ -567,7 +594,7 @@ public class TestOrcFile { assertEquals("count: 2 min: -15.0 max: -5.0 sum: -20.0", stats[7].toString()); - assertEquals("count: 2 min: bye max: hi", stats[9].toString()); + assertEquals("count: 2 min: bye max: hi sum: 5", stats[9].toString()); // check the inspectors StructObjectInspector readerInspector = Modified: hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcNullOptimization.java URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcNullOptimization.java?rev=1529347&r1=1529346&r2=1529347&view=diff ============================================================================== --- hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcNullOptimization.java (original) +++ hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcNullOptimization.java Fri Oct 4 23:35:50 2013 @@ -125,7 +125,7 @@ public class TestOrcNullOptimization { assertEquals("a", ((StringColumnStatistics) stats[2]).getMinimum()); assertEquals(19998, ((StringColumnStatistics) stats[2]).getNumberOfValues()); - assertEquals("count: 19998 min: a max: a", + assertEquals("count: 19998 min: a max: a sum: 19998", stats[2].toString()); // check the inspectors @@ -229,7 +229,7 @@ public class TestOrcNullOptimization { assertEquals("a", ((StringColumnStatistics) stats[2]).getMinimum()); assertEquals(20000, ((StringColumnStatistics) stats[2]).getNumberOfValues()); - assertEquals("count: 20000 min: a max: b", + assertEquals("count: 20000 min: a max: b sum: 20000", stats[2].toString()); // check the inspectors @@ -329,7 +329,7 @@ public class TestOrcNullOptimization { assertEquals("h", ((StringColumnStatistics) stats[2]).getMaximum()); assertEquals("a", ((StringColumnStatistics) stats[2]).getMinimum()); assertEquals(7, ((StringColumnStatistics) stats[2]).getNumberOfValues()); - assertEquals("count: 7 min: a max: h", + assertEquals("count: 7 min: a max: h sum: 7", stats[2].toString()); // check the inspectors Modified: hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveMetaStoreChecker.java URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveMetaStoreChecker.java?rev=1529347&r1=1529346&r2=1529347&view=diff ============================================================================== --- hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveMetaStoreChecker.java (original) +++ hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveMetaStoreChecker.java Fri Oct 4 23:35:50 2013 @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.metada import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -47,8 +48,8 @@ public class TestHiveMetaStoreChecker ex private FileSystem fs; private HiveMetaStoreChecker checker = null; - private final String dbName = "dbname"; - private final String tableName = "tablename"; + private final String dbName = "testhivemetastorechecker_db"; + private final String tableName = "testhivemetastorechecker_table"; private final String partDateName = "partdate"; private final String partCityName = "partcity"; @@ -76,17 +77,25 @@ public class TestHiveMetaStoreChecker ex part2.put(partCityName, "stockholm"); parts.add(part2); + //cleanup just in case something is left over from previous run + dropDbTable(); + } + + private void dropDbTable() { // cleanup - hive.dropTable(dbName, tableName, true, true); try { - hive.dropDatabase(dbName); + hive.dropTable(dbName, tableName, true, true); + hive.dropDatabase(dbName, true, true, true); } catch (NoSuchObjectException e) { // ignore + } catch (HiveException e) { + // ignore } } @Override protected void tearDown() throws Exception { + dropDbTable(); super.tearDown(); Hive.closeCurrent(); } @@ -97,19 +106,19 @@ public class TestHiveMetaStoreChecker ex CheckResult result = new CheckResult(); checker.checkMetastore(dbName, null, null, result); // we haven't added anything so should return an all ok - assertTrue(result.getTablesNotInMs().isEmpty()); - assertTrue(result.getTablesNotOnFs().isEmpty()); - assertTrue(result.getPartitionsNotOnFs().isEmpty()); - assertTrue(result.getPartitionsNotInMs().isEmpty()); + assertEquals(Collections.emptyList(), result.getTablesNotInMs()); + assertEquals(Collections.emptyList(), result.getTablesNotOnFs()); + assertEquals(Collections.emptyList(), result.getPartitionsNotOnFs()); + assertEquals(Collections.emptyList(), result.getPartitionsNotInMs()); // check table only, should not exist in ms result = new CheckResult(); checker.checkMetastore(dbName, tableName, null, result); assertEquals(1, result.getTablesNotInMs().size()); assertEquals(tableName, result.getTablesNotInMs().get(0)); - assertTrue(result.getTablesNotOnFs().isEmpty()); - assertTrue(result.getPartitionsNotOnFs().isEmpty()); - assertTrue(result.getPartitionsNotInMs().isEmpty()); + assertEquals(Collections.emptyList(), result.getTablesNotOnFs()); + assertEquals(Collections.emptyList(), result.getPartitionsNotOnFs()); + assertEquals(Collections.emptyList(), result.getPartitionsNotInMs()); Database db = new Database(); db.setName(dbName); @@ -125,18 +134,18 @@ public class TestHiveMetaStoreChecker ex // first check all (1) tables result = new CheckResult(); checker.checkMetastore(dbName, null, null, result); - assertTrue(result.getTablesNotInMs().isEmpty()); - assertTrue(result.getTablesNotOnFs().isEmpty()); - assertTrue(result.getPartitionsNotOnFs().isEmpty()); - assertTrue(result.getPartitionsNotInMs().isEmpty()); + assertEquals(Collections.emptyList(), result.getTablesNotInMs()); + assertEquals(Collections.emptyList(), result.getTablesNotOnFs()); + assertEquals(Collections.emptyList(), result.getPartitionsNotOnFs()); + assertEquals(Collections.emptyList(), result.getPartitionsNotInMs()); // then let's check the one we know about result = new CheckResult(); checker.checkMetastore(dbName, tableName, null, result); - assertTrue(result.getTablesNotInMs().isEmpty()); - assertTrue(result.getTablesNotOnFs().isEmpty()); - assertTrue(result.getPartitionsNotOnFs().isEmpty()); - assertTrue(result.getPartitionsNotInMs().isEmpty()); + assertEquals(Collections.emptyList(), result.getTablesNotInMs()); + assertEquals(Collections.emptyList(), result.getTablesNotOnFs()); + assertEquals(Collections.emptyList(), result.getPartitionsNotOnFs()); + assertEquals(Collections.emptyList(), result.getPartitionsNotInMs()); // remove the table folder fs = table.getPath().getFileSystem(hive.getConf()); @@ -145,26 +154,27 @@ public class TestHiveMetaStoreChecker ex // now this shouldn't find the path on the fs result = new CheckResult(); checker.checkMetastore(dbName, tableName, null, result); - assertTrue(result.getTablesNotInMs().isEmpty()); + assertEquals(Collections.emptyList(), result.getTablesNotInMs());; assertEquals(1, result.getTablesNotOnFs().size()); assertEquals(tableName, result.getTablesNotOnFs().get(0)); - assertTrue(result.getPartitionsNotOnFs().isEmpty()); - assertTrue(result.getPartitionsNotInMs().isEmpty()); + assertEquals(Collections.emptyList(), result.getPartitionsNotOnFs()); + assertEquals(Collections.emptyList(), result.getPartitionsNotInMs()); // put it back and one additional table fs.mkdirs(table.getPath()); Path fakeTable = table.getPath().getParent().suffix( Path.SEPARATOR + "faketable"); fs.mkdirs(fakeTable); + fs.deleteOnExit(fakeTable); // find the extra table result = new CheckResult(); checker.checkMetastore(dbName, null, null, result); assertEquals(1, result.getTablesNotInMs().size()); assertEquals(fakeTable.getName(), result.getTablesNotInMs().get(0)); - assertTrue(result.getTablesNotOnFs().isEmpty()); - assertTrue(result.getPartitionsNotOnFs().isEmpty()); - assertTrue(result.getPartitionsNotInMs().isEmpty()); + assertEquals(Collections.emptyList(), result.getTablesNotOnFs()); + assertEquals(Collections.emptyList(), result.getPartitionsNotOnFs()); + assertEquals(Collections.emptyList(), result.getPartitionsNotInMs()); // create a new external table hive.dropTable(dbName, tableName); @@ -174,10 +184,10 @@ public class TestHiveMetaStoreChecker ex // should return all ok result = new CheckResult(); checker.checkMetastore(dbName, null, null, result); - assertTrue(result.getTablesNotInMs().isEmpty()); - assertTrue(result.getTablesNotOnFs().isEmpty()); - assertTrue(result.getPartitionsNotOnFs().isEmpty()); - assertTrue(result.getPartitionsNotInMs().isEmpty()); + assertEquals(Collections.emptyList(), result.getTablesNotInMs()); + assertEquals(Collections.emptyList(), result.getTablesNotOnFs()); + assertEquals(Collections.emptyList(), result.getPartitionsNotOnFs()); + assertEquals(Collections.emptyList(), result.getPartitionsNotInMs()); } public void testPartitionsCheck() throws HiveException, MetaException, @@ -203,10 +213,10 @@ public class TestHiveMetaStoreChecker ex CheckResult result = new CheckResult(); checker.checkMetastore(dbName, tableName, null, result); // all is well - assertTrue(result.getTablesNotInMs().isEmpty()); - assertTrue(result.getTablesNotOnFs().isEmpty()); - assertTrue(result.getPartitionsNotOnFs().isEmpty()); - assertTrue(result.getPartitionsNotInMs().isEmpty()); + assertEquals(Collections.emptyList(), result.getTablesNotInMs()); + assertEquals(Collections.emptyList(), result.getTablesNotOnFs()); + assertEquals(Collections.emptyList(), result.getPartitionsNotOnFs()); + assertEquals(Collections.emptyList(), result.getPartitionsNotInMs()); List partitions = hive.getPartitions(table); assertEquals(2, partitions.size()); @@ -218,24 +228,24 @@ public class TestHiveMetaStoreChecker ex result = new CheckResult(); checker.checkMetastore(dbName, tableName, null, result); // missing one partition on fs - assertTrue(result.getTablesNotInMs().isEmpty()); - assertTrue(result.getTablesNotOnFs().isEmpty()); + assertEquals(Collections.emptyList(), result.getTablesNotInMs()); + assertEquals(Collections.emptyList(), result.getTablesNotOnFs()); assertEquals(1, result.getPartitionsNotOnFs().size()); assertEquals(partToRemove.getName(), result.getPartitionsNotOnFs().get(0) .getPartitionName()); assertEquals(partToRemove.getTable().getTableName(), result .getPartitionsNotOnFs().get(0).getTableName()); - assertTrue(result.getPartitionsNotInMs().isEmpty()); + assertEquals(Collections.emptyList(), result.getPartitionsNotInMs()); List> partsCopy = new ArrayList>(); partsCopy.add(partitions.get(1).getSpec()); // check only the partition that exists, all should be well result = new CheckResult(); checker.checkMetastore(dbName, tableName, partsCopy, result); - assertTrue(result.getTablesNotInMs().isEmpty()); - assertTrue(result.getTablesNotOnFs().isEmpty()); - assertTrue(result.getPartitionsNotOnFs().isEmpty()); - assertTrue(result.getPartitionsNotInMs().isEmpty()); + assertEquals(Collections.emptyList(), result.getTablesNotInMs()); + assertEquals(Collections.emptyList(), result.getTablesNotOnFs()); + assertEquals(Collections.emptyList(), result.getPartitionsNotOnFs()); + assertEquals(Collections.emptyList(), result.getPartitionsNotInMs()); // put the other one back fs.mkdirs(partToRemovePath); @@ -244,15 +254,60 @@ public class TestHiveMetaStoreChecker ex Path fakePart = new Path(table.getDataLocation().toString(), "fakepartition=fakevalue"); fs.mkdirs(fakePart); + fs.deleteOnExit(fakePart); checker.checkMetastore(dbName, tableName, null, result); // one extra partition - assertTrue(result.getTablesNotInMs().isEmpty()); - assertTrue(result.getTablesNotOnFs().isEmpty()); - assertTrue(result.getPartitionsNotOnFs().isEmpty()); + assertEquals(Collections.emptyList(), result.getTablesNotInMs()); + assertEquals(Collections.emptyList(), result.getTablesNotOnFs()); + assertEquals(Collections.emptyList(), result.getPartitionsNotOnFs()); assertEquals(1, result.getPartitionsNotInMs().size()); assertEquals(fakePart.getName(), result.getPartitionsNotInMs().get(0) .getPartitionName()); + + // cleanup + hive.dropTable(dbName, tableName, true, true); + hive.createTable(table); + result = new CheckResult(); + checker.checkMetastore(dbName, null, null, result); + assertEquals(Collections.emptyList(), result.getTablesNotInMs()); + assertEquals(Collections.emptyList(), result.getTablesNotOnFs()); + assertEquals(Collections.emptyList(), result.getPartitionsNotOnFs()); + assertEquals(Collections.emptyList(), result.getPartitionsNotInMs()); //--0e + System.err.println("Test completed - partition check"); + } + public void testDataDeletion() throws HiveException, MetaException, + IOException, TException, AlreadyExistsException, NoSuchObjectException { + + Database db = new Database(); + db.setName(dbName); + hive.createDatabase(db); + + Table table = new Table(dbName, tableName); + table.setDbName(dbName); + table.setInputFormatClass(TextInputFormat.class); + table.setOutputFormatClass(HiveIgnoreKeyTextOutputFormat.class); + table.setPartCols(partCols); + + hive.createTable(table); + table = hive.getTable(dbName, tableName); + + Path fakeTable = table.getPath().getParent().suffix( + Path.SEPARATOR + "faketable"); + fs = fakeTable.getFileSystem(hive.getConf()); + fs.mkdirs(fakeTable); + fs.deleteOnExit(fakeTable); + + Path fakePart = new Path(table.getDataLocation().toString(), + "fakepartition=fakevalue"); + fs.mkdirs(fakePart); + fs.deleteOnExit(fakePart); + + hive.dropTable(dbName, tableName, true, true); + assertFalse(fs.exists(fakePart)); + hive.dropDatabase(dbName); + assertFalse(fs.exists(fakeTable)); + } } Modified: hive/branches/tez/ql/src/test/queries/clientpositive/alter_varchar1.q URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/queries/clientpositive/alter_varchar1.q?rev=1529347&r1=1529346&r2=1529347&view=diff ============================================================================== --- hive/branches/tez/ql/src/test/queries/clientpositive/alter_varchar1.q (original) +++ hive/branches/tez/ql/src/test/queries/clientpositive/alter_varchar1.q Fri Oct 4 23:35:50 2013 @@ -2,31 +2,31 @@ drop table alter_varchar_1; create table alter_varchar_1 (key string, value string); insert overwrite table alter_varchar_1 - select key, value from src limit 5; + select key, value from src order by key limit 5; -select * from alter_varchar_1; +select * from alter_varchar_1 order by key; -- change column to varchar alter table alter_varchar_1 change column value value varchar(20); -- contents should still look the same -select * from alter_varchar_1; +select * from alter_varchar_1 order by key; -- change column to smaller varchar alter table alter_varchar_1 change column value value varchar(3); -- value column should be truncated now -select * from alter_varchar_1; +select * from alter_varchar_1 order by key; -- change back to bigger varchar alter table alter_varchar_1 change column value value varchar(20); -- column values should be full size again -select * from alter_varchar_1; +select * from alter_varchar_1 order by key; -- add varchar column alter table alter_varchar_1 add columns (key2 int, value2 varchar(10)); -select * from alter_varchar_1; +select * from alter_varchar_1 order by key; insert overwrite table alter_varchar_1 - select key, value, key, value from src limit 5; -select * from alter_varchar_1; + select key, value, key, value from src order by key limit 5; +select * from alter_varchar_1 order by key; drop table alter_varchar_1; Modified: hive/branches/tez/ql/src/test/queries/clientpositive/orc_create.q URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/queries/clientpositive/orc_create.q?rev=1529347&r1=1529346&r2=1529347&view=diff ============================================================================== --- hive/branches/tez/ql/src/test/queries/clientpositive/orc_create.q (original) +++ hive/branches/tez/ql/src/test/queries/clientpositive/orc_create.q Fri Oct 4 23:35:50 2013 @@ -1,6 +1,8 @@ DROP TABLE orc_create; DROP TABLE orc_create_complex; DROP TABLE orc_create_staging; +DROP TABLE orc_create_people_staging; +DROP TABLE orc_create_people; CREATE TABLE orc_create_staging ( str STRING, @@ -38,6 +40,8 @@ set hive.default.fileformat=orc; CREATE TABLE orc_create (key INT, value STRING) PARTITIONED BY (ds string); +set hive.default.fileformat=text; + DESCRIBE FORMATTED orc_create; CREATE TABLE orc_create_complex ( @@ -61,6 +65,39 @@ SELECT mp from orc_create_complex; SELECT lst from orc_create_complex; SELECT strct from orc_create_complex; +CREATE TABLE orc_create_people_staging ( + id int, + first_name string, + last_name string, + address string, + state string); + +LOAD DATA LOCAL INPATH '../data/files/orc_create_people.txt' + OVERWRITE INTO TABLE orc_create_people_staging; + +CREATE TABLE orc_create_people ( + id int, + first_name string, + last_name string, + address string) +PARTITIONED BY (state string) +STORED AS orc; + +set hive.exec.dynamic.partition.mode=nonstrict; + +INSERT OVERWRITE TABLE orc_create_people PARTITION (state) + SELECT * FROM orc_create_people_staging; + +SET hive.optimize.index.filter=true; +-- test predicate push down with partition pruning +SELECT COUNT(*) FROM orc_create_people where id < 10 and state = 'Ca'; + +-- test predicate push down with no column projection +SELECT id, first_name, last_name, address + FROM orc_create_people WHERE id > 90; + DROP TABLE orc_create; DROP TABLE orc_create_complex; DROP TABLE orc_create_staging; +DROP TABLE orc_create_people_staging; +DROP TABLE orc_create_people; Modified: hive/branches/tez/ql/src/test/queries/clientpositive/stats_partscan_1.q URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/queries/clientpositive/stats_partscan_1.q?rev=1529347&r1=1529346&r2=1529347&view=diff ============================================================================== --- hive/branches/tez/ql/src/test/queries/clientpositive/stats_partscan_1.q (original) +++ hive/branches/tez/ql/src/test/queries/clientpositive/stats_partscan_1.q Fri Oct 4 23:35:50 2013 @@ -7,6 +7,11 @@ set mapred.min.split.size.per.node=256; set mapred.min.split.size.per.rack=256; set mapred.max.split.size=256; +-- INCLUDE_HADOOP_MAJOR_VERSIONS(0.20,0.20S) +-- This test uses mapred.max.split.size/mapred.max.split.size for controlling +-- number of input splits, which is not effective in hive 0.20. +-- stats_partscan_1_23.q is the same test with this but has different result. + -- test analyze table ... compute statistics partialscan -- 1. prepare data Modified: hive/branches/tez/ql/src/test/queries/clientpositive/varchar_1.q URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/queries/clientpositive/varchar_1.q?rev=1529347&r1=1529346&r2=1529347&view=diff ============================================================================== --- hive/branches/tez/ql/src/test/queries/clientpositive/varchar_1.q (original) +++ hive/branches/tez/ql/src/test/queries/clientpositive/varchar_1.q Fri Oct 4 23:35:50 2013 @@ -6,27 +6,27 @@ create table varchar1_1 (key string, val -- load from file load data local inpath '../data/files/srcbucket0.txt' overwrite into table varchar1; -select * from varchar1 limit 2; +select * from varchar1 order by key, value limit 2; -- insert overwrite, from same/different length varchar insert overwrite table varchar1 - select cast(key as varchar(10)), cast(value as varchar(15)) from src limit 2; -select key, value from varchar1; + select cast(key as varchar(10)), cast(value as varchar(15)) from src order by key, value limit 2; +select key, value from varchar1 order by key, value; -- insert overwrite, from string insert overwrite table varchar1 - select key, value from src limit 2; -select key, value from varchar1; + select key, value from src order by key, value limit 2; +select key, value from varchar1 order by key, value; -- insert string from varchar insert overwrite table varchar1_1 - select key, value from varchar1 limit 2; -select key, value from varchar1_1; + select key, value from varchar1 order by key, value limit 2; +select key, value from varchar1_1 order by key, value; -- respect string length insert overwrite table varchar1 - select key, cast(value as varchar(3)) from src limit 2; -select key, value from varchar1; + select key, cast(value as varchar(3)) from src order by key, value limit 2; +select key, value from varchar1 order by key, value; drop table varchar1; drop table varchar1_1; Modified: hive/branches/tez/ql/src/test/queries/clientpositive/varchar_nested_types.q URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/queries/clientpositive/varchar_nested_types.q?rev=1529347&r1=1529346&r2=1529347&view=diff ============================================================================== --- hive/branches/tez/ql/src/test/queries/clientpositive/varchar_nested_types.q (original) +++ hive/branches/tez/ql/src/test/queries/clientpositive/varchar_nested_types.q Fri Oct 4 23:35:50 2013 @@ -7,7 +7,7 @@ drop table varchar_nested_view; create table varchar_nested_1 (key int, value varchar(20)); insert overwrite table varchar_nested_1 - select key, value from src limit 5; + select key, value from src order by key limit 1; -- arrays create table varchar_nested_array (c1 array); Modified: hive/branches/tez/ql/src/test/queries/clientpositive/varchar_udf1.q URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/queries/clientpositive/varchar_udf1.q?rev=1529347&r1=1529346&r2=1529347&view=diff ============================================================================== --- hive/branches/tez/ql/src/test/queries/clientpositive/varchar_udf1.q (original) +++ hive/branches/tez/ql/src/test/queries/clientpositive/varchar_udf1.q Fri Oct 4 23:35:50 2013 @@ -2,7 +2,7 @@ drop table varchar_udf_1; create table varchar_udf_1 (c1 string, c2 string, c3 varchar(10), c4 varchar(20)); insert overwrite table varchar_udf_1 - select key, value, key, value from src limit 5; + select key, value, key, value from src where key = '238' limit 1; -- UDFs with varchar support select Modified: hive/branches/tez/ql/src/test/resources/orc-file-dump-dictionary-threshold.out URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/resources/orc-file-dump-dictionary-threshold.out?rev=1529347&r1=1529346&r2=1529347&view=diff ============================================================================== --- hive/branches/tez/ql/src/test/resources/orc-file-dump-dictionary-threshold.out (original) +++ hive/branches/tez/ql/src/test/resources/orc-file-dump-dictionary-threshold.out Fri Oct 4 23:35:50 2013 @@ -8,71 +8,71 @@ Statistics: Column 0: count: 21000 Column 1: count: 21000 min: -2147390285 max: 2147453086 sum: 109128518326 Column 2: count: 21000 min: -9222731174895935707 max: 9222919052987871506 - Column 3: count: 21000 min: Darkness,-230 max: worst-54-290-346-648-908-996-1038-1080-1560-1584-1620-1744-1770-1798-1852-1966-2162-2244-2286-2296-2534-2660-3114-3676-3788-4068-4150-4706-4744-5350-5420-5582-5696-5726-6006-6020-6024-6098-6184-6568-6636-6802-6994-7004-7318-7498-7758-7780-7798-7920-7952-7960-7988-8232-8256-8390-8416-8478-8620-8840-8984-9038-9128-9236-9248-9344-9594-9650-9714-9928-9938-10178-10368-10414-10502-10732-10876-11008-11158-11410-11722-11836-11964-12054-12096-12126-12136-12202-12246-12298-12616-12774-12782-12790-12802-12976-13216-13246-13502-13766-14454-14974-15004-15124-15252-15294-15356-15530-15610-16316-16936-17024-17122-17214-17310-17528-17682-17742-17870-17878-18010-18410-18524-18788-19204-19254-19518-19596-19786-19874-19904-20390-20752-20936 + Column 3: count: 21000 min: Darkness,-230 max: worst-54-290-346-648-908-996-1038-1080-1560-1584-1620-1744-1770-1798-1852-1966-2162-2244-2286-2296-2534-2660-3114-3676-3788-4068-4150-4706-4744-5350-5420-5582-5696-5726-6006-6020-6024-6098-6184-6568-6636-6802-6994-7004-7318-7498-7758-7780-7798-7920-7952-7960-7988-8232-8256-8390-8416-8478-8620-8840-8984-9038-9128-9236-9248-9344-9594-9650-9714-9928-9938-10178-10368-10414-10502-10732-10876-11008-11158-11410-11722-11836-11964-12054-12096-12126-12136-12202-12246-12298-12616-12774-12782-12790-12802-12976-13216-13246-13502-13766-14454-14974-15004-15124-15252-15294-15356-15530-15610-16316-16936-17024-17122-17214-17310-17528-17682-17742-17870-17878-18010-18410-18524-18788-19204-19254-19518-19596-19786-19874-19904-20390-20752-20936 sum: 6910238 Stripes: - Stripe: offset: 3 data: 102311 rows: 4000 tail: 68 index: 217 + Stripe: offset: 3 data: 102311 rows: 4000 tail: 68 index: 224 Stream: column 0 section ROW_INDEX start: 3 length 10 Stream: column 1 section ROW_INDEX start: 13 length 36 Stream: column 2 section ROW_INDEX start: 49 length 39 - Stream: column 3 section ROW_INDEX start: 88 length 132 - Stream: column 1 section DATA start: 220 length 16022 - Stream: column 2 section DATA start: 16242 length 32028 - Stream: column 3 section DATA start: 48270 length 50887 - Stream: column 3 section LENGTH start: 99157 length 3374 + Stream: column 3 section ROW_INDEX start: 88 length 139 + Stream: column 1 section DATA start: 227 length 16022 + Stream: column 2 section DATA start: 16249 length 32028 + Stream: column 3 section DATA start: 48277 length 50887 + Stream: column 3 section LENGTH start: 99164 length 3374 Encoding column 0: DIRECT Encoding column 1: DIRECT_V2 Encoding column 2: DIRECT_V2 Encoding column 3: DIRECT_V2 - Stripe: offset: 102599 data: 284999 rows: 5000 tail: 68 index: 349 - Stream: column 0 section ROW_INDEX start: 102599 length 10 - Stream: column 1 section ROW_INDEX start: 102609 length 36 - Stream: column 2 section ROW_INDEX start: 102645 length 39 - Stream: column 3 section ROW_INDEX start: 102684 length 264 - Stream: column 1 section DATA start: 102948 length 20029 - Stream: column 2 section DATA start: 122977 length 40035 - Stream: column 3 section DATA start: 163012 length 219588 - Stream: column 3 section LENGTH start: 382600 length 5347 + Stripe: offset: 102606 data: 284999 rows: 5000 tail: 68 index: 356 + Stream: column 0 section ROW_INDEX start: 102606 length 10 + Stream: column 1 section ROW_INDEX start: 102616 length 36 + Stream: column 2 section ROW_INDEX start: 102652 length 39 + Stream: column 3 section ROW_INDEX start: 102691 length 271 + Stream: column 1 section DATA start: 102962 length 20029 + Stream: column 2 section DATA start: 122991 length 40035 + Stream: column 3 section DATA start: 163026 length 219588 + Stream: column 3 section LENGTH start: 382614 length 5347 Encoding column 0: DIRECT Encoding column 1: DIRECT_V2 Encoding column 2: DIRECT_V2 Encoding column 3: DIRECT_V2 - Stripe: offset: 388015 data: 491655 rows: 5000 tail: 69 index: 536 - Stream: column 0 section ROW_INDEX start: 388015 length 10 - Stream: column 1 section ROW_INDEX start: 388025 length 36 - Stream: column 2 section ROW_INDEX start: 388061 length 39 - Stream: column 3 section ROW_INDEX start: 388100 length 451 - Stream: column 1 section DATA start: 388551 length 20029 - Stream: column 2 section DATA start: 408580 length 40035 - Stream: column 3 section DATA start: 448615 length 425862 - Stream: column 3 section LENGTH start: 874477 length 5729 + Stripe: offset: 388029 data: 491655 rows: 5000 tail: 69 index: 544 + Stream: column 0 section ROW_INDEX start: 388029 length 10 + Stream: column 1 section ROW_INDEX start: 388039 length 36 + Stream: column 2 section ROW_INDEX start: 388075 length 39 + Stream: column 3 section ROW_INDEX start: 388114 length 459 + Stream: column 1 section DATA start: 388573 length 20029 + Stream: column 2 section DATA start: 408602 length 40035 + Stream: column 3 section DATA start: 448637 length 425862 + Stream: column 3 section LENGTH start: 874499 length 5729 Encoding column 0: DIRECT Encoding column 1: DIRECT_V2 Encoding column 2: DIRECT_V2 Encoding column 3: DIRECT_V2 - Stripe: offset: 880275 data: 707368 rows: 5000 tail: 68 index: 677 - Stream: column 0 section ROW_INDEX start: 880275 length 10 - Stream: column 1 section ROW_INDEX start: 880285 length 36 - Stream: column 2 section ROW_INDEX start: 880321 length 39 - Stream: column 3 section ROW_INDEX start: 880360 length 592 - Stream: column 1 section DATA start: 880952 length 20029 - Stream: column 2 section DATA start: 900981 length 40035 - Stream: column 3 section DATA start: 941016 length 641580 - Stream: column 3 section LENGTH start: 1582596 length 5724 + Stripe: offset: 880297 data: 707368 rows: 5000 tail: 68 index: 691 + Stream: column 0 section ROW_INDEX start: 880297 length 10 + Stream: column 1 section ROW_INDEX start: 880307 length 36 + Stream: column 2 section ROW_INDEX start: 880343 length 39 + Stream: column 3 section ROW_INDEX start: 880382 length 606 + Stream: column 1 section DATA start: 880988 length 20029 + Stream: column 2 section DATA start: 901017 length 40035 + Stream: column 3 section DATA start: 941052 length 641580 + Stream: column 3 section LENGTH start: 1582632 length 5724 Encoding column 0: DIRECT Encoding column 1: DIRECT_V2 Encoding column 2: DIRECT_V2 Encoding column 3: DIRECT_V2 - Stripe: offset: 1588388 data: 348697 rows: 2000 tail: 67 index: 786 - Stream: column 0 section ROW_INDEX start: 1588388 length 10 - Stream: column 1 section ROW_INDEX start: 1588398 length 36 - Stream: column 2 section ROW_INDEX start: 1588434 length 39 - Stream: column 3 section ROW_INDEX start: 1588473 length 701 - Stream: column 1 section DATA start: 1589174 length 8011 - Stream: column 2 section DATA start: 1597185 length 16014 - Stream: column 3 section DATA start: 1613199 length 322259 - Stream: column 3 section LENGTH start: 1935458 length 2413 + Stripe: offset: 1588424 data: 348697 rows: 2000 tail: 67 index: 797 + Stream: column 0 section ROW_INDEX start: 1588424 length 10 + Stream: column 1 section ROW_INDEX start: 1588434 length 36 + Stream: column 2 section ROW_INDEX start: 1588470 length 39 + Stream: column 3 section ROW_INDEX start: 1588509 length 712 + Stream: column 1 section DATA start: 1589221 length 8011 + Stream: column 2 section DATA start: 1597232 length 16014 + Stream: column 3 section DATA start: 1613246 length 322259 + Stream: column 3 section LENGTH start: 1935505 length 2413 Encoding column 0: DIRECT Encoding column 1: DIRECT_V2 Encoding column 2: DIRECT_V2 - Encoding column 3: DIRECT_V2 \ No newline at end of file + Encoding column 3: DIRECT_V2 Modified: hive/branches/tez/ql/src/test/resources/orc-file-dump.out URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/resources/orc-file-dump.out?rev=1529347&r1=1529346&r2=1529347&view=diff ============================================================================== --- hive/branches/tez/ql/src/test/resources/orc-file-dump.out (original) +++ hive/branches/tez/ql/src/test/resources/orc-file-dump.out Fri Oct 4 23:35:50 2013 @@ -8,75 +8,75 @@ Statistics: Column 0: count: 21000 Column 1: count: 21000 min: -2146993718 max: 2147378179 sum: 193017464403 Column 2: count: 21000 min: -9222758097219661129 max: 9222303228623055266 - Column 3: count: 21000 min: Darkness, max: worst + Column 3: count: 21000 min: Darkness, max: worst sum: 81761 Stripes: - Stripe: offset: 3 data: 63766 rows: 5000 tail: 74 index: 119 + Stripe: offset: 3 data: 63766 rows: 5000 tail: 74 index: 123 Stream: column 0 section ROW_INDEX start: 3 length 10 Stream: column 1 section ROW_INDEX start: 13 length 35 Stream: column 2 section ROW_INDEX start: 48 length 39 - Stream: column 3 section ROW_INDEX start: 87 length 35 - Stream: column 1 section DATA start: 122 length 20029 - Stream: column 2 section DATA start: 20151 length 40035 - Stream: column 3 section DATA start: 60186 length 3544 - Stream: column 3 section LENGTH start: 63730 length 25 - Stream: column 3 section DICTIONARY_DATA start: 63755 length 133 + Stream: column 3 section ROW_INDEX start: 87 length 39 + Stream: column 1 section DATA start: 126 length 20029 + Stream: column 2 section DATA start: 20155 length 40035 + Stream: column 3 section DATA start: 60190 length 3544 + Stream: column 3 section LENGTH start: 63734 length 25 + Stream: column 3 section DICTIONARY_DATA start: 63759 length 133 Encoding column 0: DIRECT Encoding column 1: DIRECT_V2 Encoding column 2: DIRECT_V2 Encoding column 3: DICTIONARY_V2 - Stripe: offset: 63962 data: 63755 rows: 5000 tail: 76 index: 118 - Stream: column 0 section ROW_INDEX start: 63962 length 10 - Stream: column 1 section ROW_INDEX start: 63972 length 34 - Stream: column 2 section ROW_INDEX start: 64006 length 39 - Stream: column 3 section ROW_INDEX start: 64045 length 35 - Stream: column 1 section DATA start: 64080 length 20029 - Stream: column 2 section DATA start: 84109 length 40035 - Stream: column 3 section DATA start: 124144 length 3533 - Stream: column 3 section LENGTH start: 127677 length 25 - Stream: column 3 section DICTIONARY_DATA start: 127702 length 133 + Stripe: offset: 63966 data: 63755 rows: 5000 tail: 74 index: 122 + Stream: column 0 section ROW_INDEX start: 63966 length 10 + Stream: column 1 section ROW_INDEX start: 63976 length 34 + Stream: column 2 section ROW_INDEX start: 64010 length 39 + Stream: column 3 section ROW_INDEX start: 64049 length 39 + Stream: column 1 section DATA start: 64088 length 20029 + Stream: column 2 section DATA start: 84117 length 40035 + Stream: column 3 section DATA start: 124152 length 3533 + Stream: column 3 section LENGTH start: 127685 length 25 + Stream: column 3 section DICTIONARY_DATA start: 127710 length 133 Encoding column 0: DIRECT Encoding column 1: DIRECT_V2 Encoding column 2: DIRECT_V2 Encoding column 3: DICTIONARY_V2 - Stripe: offset: 127911 data: 63766 rows: 5000 tail: 76 index: 120 - Stream: column 0 section ROW_INDEX start: 127911 length 10 - Stream: column 1 section ROW_INDEX start: 127921 length 36 - Stream: column 2 section ROW_INDEX start: 127957 length 39 - Stream: column 3 section ROW_INDEX start: 127996 length 35 - Stream: column 1 section DATA start: 128031 length 20029 - Stream: column 2 section DATA start: 148060 length 40035 - Stream: column 3 section DATA start: 188095 length 3544 - Stream: column 3 section LENGTH start: 191639 length 25 - Stream: column 3 section DICTIONARY_DATA start: 191664 length 133 + Stripe: offset: 127917 data: 63766 rows: 5000 tail: 74 index: 124 + Stream: column 0 section ROW_INDEX start: 127917 length 10 + Stream: column 1 section ROW_INDEX start: 127927 length 36 + Stream: column 2 section ROW_INDEX start: 127963 length 39 + Stream: column 3 section ROW_INDEX start: 128002 length 39 + Stream: column 1 section DATA start: 128041 length 20029 + Stream: column 2 section DATA start: 148070 length 40035 + Stream: column 3 section DATA start: 188105 length 3544 + Stream: column 3 section LENGTH start: 191649 length 25 + Stream: column 3 section DICTIONARY_DATA start: 191674 length 133 Encoding column 0: DIRECT Encoding column 1: DIRECT_V2 Encoding column 2: DIRECT_V2 Encoding column 3: DICTIONARY_V2 - Stripe: offset: 200000 data: 63796 rows: 5000 tail: 74 index: 119 + Stripe: offset: 200000 data: 63796 rows: 5000 tail: 74 index: 123 Stream: column 0 section ROW_INDEX start: 200000 length 10 Stream: column 1 section ROW_INDEX start: 200010 length 35 Stream: column 2 section ROW_INDEX start: 200045 length 39 - Stream: column 3 section ROW_INDEX start: 200084 length 35 - Stream: column 1 section DATA start: 200119 length 20029 - Stream: column 2 section DATA start: 220148 length 40035 - Stream: column 3 section DATA start: 260183 length 3574 - Stream: column 3 section LENGTH start: 263757 length 25 - Stream: column 3 section DICTIONARY_DATA start: 263782 length 133 + Stream: column 3 section ROW_INDEX start: 200084 length 39 + Stream: column 1 section DATA start: 200123 length 20029 + Stream: column 2 section DATA start: 220152 length 40035 + Stream: column 3 section DATA start: 260187 length 3574 + Stream: column 3 section LENGTH start: 263761 length 25 + Stream: column 3 section DICTIONARY_DATA start: 263786 length 133 Encoding column 0: DIRECT Encoding column 1: DIRECT_V2 Encoding column 2: DIRECT_V2 Encoding column 3: DICTIONARY_V2 - Stripe: offset: 263989 data: 12940 rows: 1000 tail: 71 index: 120 - Stream: column 0 section ROW_INDEX start: 263989 length 10 - Stream: column 1 section ROW_INDEX start: 263999 length 36 - Stream: column 2 section ROW_INDEX start: 264035 length 39 - Stream: column 3 section ROW_INDEX start: 264074 length 35 - Stream: column 1 section DATA start: 264109 length 4007 - Stream: column 2 section DATA start: 268116 length 8007 - Stream: column 3 section DATA start: 276123 length 768 - Stream: column 3 section LENGTH start: 276891 length 25 - Stream: column 3 section DICTIONARY_DATA start: 276916 length 133 + Stripe: offset: 263993 data: 12940 rows: 1000 tail: 71 index: 123 + Stream: column 0 section ROW_INDEX start: 263993 length 10 + Stream: column 1 section ROW_INDEX start: 264003 length 36 + Stream: column 2 section ROW_INDEX start: 264039 length 39 + Stream: column 3 section ROW_INDEX start: 264078 length 38 + Stream: column 1 section DATA start: 264116 length 4007 + Stream: column 2 section DATA start: 268123 length 8007 + Stream: column 3 section DATA start: 276130 length 768 + Stream: column 3 section LENGTH start: 276898 length 25 + Stream: column 3 section DICTIONARY_DATA start: 276923 length 133 Encoding column 0: DIRECT Encoding column 1: DIRECT_V2 Encoding column 2: DIRECT_V2 Modified: hive/branches/tez/ql/src/test/results/clientpositive/alter_varchar1.q.out URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/results/clientpositive/alter_varchar1.q.out?rev=1529347&r1=1529346&r2=1529347&view=diff ============================================================================== --- hive/branches/tez/ql/src/test/results/clientpositive/alter_varchar1.q.out (original) +++ hive/branches/tez/ql/src/test/results/clientpositive/alter_varchar1.q.out Fri Oct 4 23:35:50 2013 @@ -8,32 +8,32 @@ POSTHOOK: query: create table alter_varc POSTHOOK: type: CREATETABLE POSTHOOK: Output: default@alter_varchar_1 PREHOOK: query: insert overwrite table alter_varchar_1 - select key, value from src limit 5 + select key, value from src order by key limit 5 PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Output: default@alter_varchar_1 POSTHOOK: query: insert overwrite table alter_varchar_1 - select key, value from src limit 5 + select key, value from src order by key limit 5 POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Output: default@alter_varchar_1 POSTHOOK: Lineage: alter_varchar_1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: alter_varchar_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: select * from alter_varchar_1 +PREHOOK: query: select * from alter_varchar_1 order by key PREHOOK: type: QUERY PREHOOK: Input: default@alter_varchar_1 #### A masked pattern was here #### -POSTHOOK: query: select * from alter_varchar_1 +POSTHOOK: query: select * from alter_varchar_1 order by key POSTHOOK: type: QUERY POSTHOOK: Input: default@alter_varchar_1 #### A masked pattern was here #### POSTHOOK: Lineage: alter_varchar_1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: alter_varchar_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -238 val_238 -86 val_86 -311 val_311 -27 val_27 -165 val_165 +0 val_0 +0 val_0 +0 val_0 +10 val_10 +100 val_100 PREHOOK: query: -- change column to varchar alter table alter_varchar_1 change column value value varchar(20) PREHOOK: type: ALTERTABLE_RENAMECOL @@ -47,22 +47,22 @@ POSTHOOK: Output: default@alter_varchar_ POSTHOOK: Lineage: alter_varchar_1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: alter_varchar_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: -- contents should still look the same -select * from alter_varchar_1 +select * from alter_varchar_1 order by key PREHOOK: type: QUERY PREHOOK: Input: default@alter_varchar_1 #### A masked pattern was here #### POSTHOOK: query: -- contents should still look the same -select * from alter_varchar_1 +select * from alter_varchar_1 order by key POSTHOOK: type: QUERY POSTHOOK: Input: default@alter_varchar_1 #### A masked pattern was here #### POSTHOOK: Lineage: alter_varchar_1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: alter_varchar_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -238 val_238 -86 val_86 -311 val_311 -27 val_27 -165 val_165 +0 val_0 +0 val_0 +0 val_0 +10 val_10 +100 val_100 PREHOOK: query: -- change column to smaller varchar alter table alter_varchar_1 change column value value varchar(3) PREHOOK: type: ALTERTABLE_RENAMECOL @@ -76,22 +76,22 @@ POSTHOOK: Output: default@alter_varchar_ POSTHOOK: Lineage: alter_varchar_1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: alter_varchar_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: -- value column should be truncated now -select * from alter_varchar_1 +select * from alter_varchar_1 order by key PREHOOK: type: QUERY PREHOOK: Input: default@alter_varchar_1 #### A masked pattern was here #### POSTHOOK: query: -- value column should be truncated now -select * from alter_varchar_1 +select * from alter_varchar_1 order by key POSTHOOK: type: QUERY POSTHOOK: Input: default@alter_varchar_1 #### A masked pattern was here #### POSTHOOK: Lineage: alter_varchar_1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: alter_varchar_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -238 val -86 val -311 val -27 val -165 val +0 val +0 val +0 val +10 val +100 val PREHOOK: query: -- change back to bigger varchar alter table alter_varchar_1 change column value value varchar(20) PREHOOK: type: ALTERTABLE_RENAMECOL @@ -105,22 +105,22 @@ POSTHOOK: Output: default@alter_varchar_ POSTHOOK: Lineage: alter_varchar_1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: alter_varchar_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: -- column values should be full size again -select * from alter_varchar_1 +select * from alter_varchar_1 order by key PREHOOK: type: QUERY PREHOOK: Input: default@alter_varchar_1 #### A masked pattern was here #### POSTHOOK: query: -- column values should be full size again -select * from alter_varchar_1 +select * from alter_varchar_1 order by key POSTHOOK: type: QUERY POSTHOOK: Input: default@alter_varchar_1 #### A masked pattern was here #### POSTHOOK: Lineage: alter_varchar_1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: alter_varchar_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -238 val_238 -86 val_86 -311 val_311 -27 val_27 -165 val_165 +0 val_0 +0 val_0 +0 val_0 +10 val_10 +100 val_100 PREHOOK: query: -- add varchar column alter table alter_varchar_1 add columns (key2 int, value2 varchar(10)) PREHOOK: type: ALTERTABLE_ADDCOLS @@ -133,28 +133,28 @@ POSTHOOK: Input: default@alter_varchar_1 POSTHOOK: Output: default@alter_varchar_1 POSTHOOK: Lineage: alter_varchar_1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: alter_varchar_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: select * from alter_varchar_1 +PREHOOK: query: select * from alter_varchar_1 order by key PREHOOK: type: QUERY PREHOOK: Input: default@alter_varchar_1 #### A masked pattern was here #### -POSTHOOK: query: select * from alter_varchar_1 +POSTHOOK: query: select * from alter_varchar_1 order by key POSTHOOK: type: QUERY POSTHOOK: Input: default@alter_varchar_1 #### A masked pattern was here #### POSTHOOK: Lineage: alter_varchar_1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: alter_varchar_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -238 val_238 NULL NULL -86 val_86 NULL NULL -311 val_311 NULL NULL -27 val_27 NULL NULL -165 val_165 NULL NULL +0 val_0 NULL NULL +0 val_0 NULL NULL +0 val_0 NULL NULL +10 val_10 NULL NULL +100 val_100 NULL NULL PREHOOK: query: insert overwrite table alter_varchar_1 - select key, value, key, value from src limit 5 + select key, value, key, value from src order by key limit 5 PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Output: default@alter_varchar_1 POSTHOOK: query: insert overwrite table alter_varchar_1 - select key, value, key, value from src limit 5 + select key, value, key, value from src order by key limit 5 POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Output: default@alter_varchar_1 @@ -164,11 +164,11 @@ POSTHOOK: Lineage: alter_varchar_1.key2 POSTHOOK: Lineage: alter_varchar_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: alter_varchar_1.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: alter_varchar_1.value2 EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: select * from alter_varchar_1 +PREHOOK: query: select * from alter_varchar_1 order by key PREHOOK: type: QUERY PREHOOK: Input: default@alter_varchar_1 #### A masked pattern was here #### -POSTHOOK: query: select * from alter_varchar_1 +POSTHOOK: query: select * from alter_varchar_1 order by key POSTHOOK: type: QUERY POSTHOOK: Input: default@alter_varchar_1 #### A masked pattern was here #### @@ -178,11 +178,11 @@ POSTHOOK: Lineage: alter_varchar_1.key2 POSTHOOK: Lineage: alter_varchar_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: alter_varchar_1.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: alter_varchar_1.value2 EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] -238 val_238 238 val_238 -86 val_86 86 val_86 -311 val_311 311 val_311 -27 val_27 27 val_27 -165 val_165 165 val_165 +0 val_0 0 val_0 +0 val_0 0 val_0 +0 val_0 0 val_0 +10 val_10 10 val_10 +100 val_100 100 val_100 PREHOOK: query: drop table alter_varchar_1 PREHOOK: type: DROPTABLE PREHOOK: Input: default@alter_varchar_1 Modified: hive/branches/tez/ql/src/test/results/clientpositive/combine2.q.out URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/results/clientpositive/combine2.q.out?rev=1529347&r1=1529346&r2=1529347&view=diff ============================================================================== --- hive/branches/tez/ql/src/test/results/clientpositive/combine2.q.out (original) +++ hive/branches/tez/ql/src/test/results/clientpositive/combine2.q.out Fri Oct 4 23:35:50 2013 @@ -142,6 +142,7 @@ STAGE PLANS: table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -634,6 +635,8 @@ STAGE PLANS: escape.delim \ hive.serialization.extend.nesting.levels true serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe TotalFiles: 1 GatherStats: false MultiFileSpray: false @@ -753,6 +756,7 @@ STAGE PLANS: table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator Modified: hive/branches/tez/ql/src/test/results/clientpositive/ctas.q.out URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/results/clientpositive/ctas.q.out?rev=1529347&r1=1529346&r2=1529347&view=diff ============================================================================== --- hive/branches/tez/ql/src/test/results/clientpositive/ctas.q.out (original) +++ hive/branches/tez/ql/src/test/results/clientpositive/ctas.q.out Fri Oct 4 23:35:50 2013 @@ -65,11 +65,13 @@ STAGE PLANS: table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: #### A masked pattern was here #### + TableScan Reduce Output Operator key expressions: expr: _col0 @@ -92,6 +94,7 @@ STAGE PLANS: table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.nzhang_CTAS1 Stage: Stage-0 @@ -224,11 +227,13 @@ STAGE PLANS: table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: #### A masked pattern was here #### + TableScan Reduce Output Operator key expressions: expr: _col0 @@ -251,6 +256,7 @@ STAGE PLANS: table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.nzhang_ctas2 Stage: Stage-0 @@ -383,11 +389,13 @@ STAGE PLANS: table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: #### A masked pattern was here #### + TableScan Reduce Output Operator key expressions: expr: _col0 @@ -410,6 +418,7 @@ STAGE PLANS: table: input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.nzhang_ctas3 Stage: Stage-0 @@ -610,11 +619,13 @@ STAGE PLANS: table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: #### A masked pattern was here #### + TableScan Reduce Output Operator key expressions: expr: _col0 @@ -637,6 +648,7 @@ STAGE PLANS: table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.nzhang_ctas4 Stage: Stage-0 @@ -826,6 +838,8 @@ STAGE PLANS: columns _col0,_col1 columns.types string,string escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe TotalFiles: 1 GatherStats: false MultiFileSpray: false @@ -834,6 +848,8 @@ STAGE PLANS: Map Reduce Alias -> Map Operator Tree: #### A masked pattern was here #### + TableScan + GatherStats: false Reduce Output Operator key expressions: expr: _col0 @@ -859,6 +875,8 @@ STAGE PLANS: columns _col0,_col1 columns.types string,string escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -866,6 +884,8 @@ STAGE PLANS: columns _col0,_col1 columns.types string,string escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Truncated Path -> Alias: #### A masked pattern was here #### Needs Tagging: false @@ -889,6 +909,8 @@ STAGE PLANS: name default.nzhang_ctas5 serialization.format , + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.nzhang_ctas5 TotalFiles: 1 GatherStats: true Modified: hive/branches/tez/ql/src/test/results/clientpositive/input12.q.out URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/results/clientpositive/input12.q.out?rev=1529347&r1=1529346&r2=1529347&view=diff ============================================================================== --- hive/branches/tez/ql/src/test/results/clientpositive/input12.q.out (original) +++ hive/branches/tez/ql/src/test/results/clientpositive/input12.q.out Fri Oct 4 23:35:50 2013 @@ -145,6 +145,7 @@ STAGE PLANS: Map Reduce Alias -> Map Operator Tree: #### A masked pattern was here #### + TableScan File Output Operator compressed: false GlobalTableId: 0 @@ -158,6 +159,7 @@ STAGE PLANS: Map Reduce Alias -> Map Operator Tree: #### A masked pattern was here #### + TableScan File Output Operator compressed: false GlobalTableId: 0 @@ -199,6 +201,7 @@ STAGE PLANS: Map Reduce Alias -> Map Operator Tree: #### A masked pattern was here #### + TableScan File Output Operator compressed: false GlobalTableId: 0 @@ -212,6 +215,7 @@ STAGE PLANS: Map Reduce Alias -> Map Operator Tree: #### A masked pattern was here #### + TableScan File Output Operator compressed: false GlobalTableId: 0 @@ -256,6 +260,7 @@ STAGE PLANS: Map Reduce Alias -> Map Operator Tree: #### A masked pattern was here #### + TableScan File Output Operator compressed: false GlobalTableId: 0 @@ -269,6 +274,7 @@ STAGE PLANS: Map Reduce Alias -> Map Operator Tree: #### A masked pattern was here #### + TableScan File Output Operator compressed: false GlobalTableId: 0 Modified: hive/branches/tez/ql/src/test/results/clientpositive/input39.q.out URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/results/clientpositive/input39.q.out?rev=1529347&r1=1529346&r2=1529347&view=diff ============================================================================== --- hive/branches/tez/ql/src/test/results/clientpositive/input39.q.out (original) +++ hive/branches/tez/ql/src/test/results/clientpositive/input39.q.out Fri Oct 4 23:35:50 2013 @@ -133,11 +133,13 @@ STAGE PLANS: table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: #### A masked pattern was here #### + TableScan Reduce Output Operator sort order: tag: -1 @@ -162,6 +164,7 @@ STAGE PLANS: table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator Modified: hive/branches/tez/ql/src/test/results/clientpositive/list_bucket_dml_1.q.out URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/results/clientpositive/list_bucket_dml_1.q.out?rev=1529347&r1=1529346&r2=1529347&view=diff ============================================================================== --- hive/branches/tez/ql/src/test/results/clientpositive/list_bucket_dml_1.q.out (original) +++ hive/branches/tez/ql/src/test/results/clientpositive/list_bucket_dml_1.q.out Fri Oct 4 23:35:50 2013 @@ -424,6 +424,8 @@ STAGE PLANS: escape.delim \ hive.serialization.extend.nesting.levels true serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe TotalFiles: 1 GatherStats: false MultiFileSpray: false