Return-Path: Delivered-To: apmail-hadoop-core-commits-archive@www.apache.org Received: (qmail 40965 invoked from network); 21 Oct 2008 18:12:58 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.2) by minotaur.apache.org with SMTP; 21 Oct 2008 18:12:58 -0000 Received: (qmail 46321 invoked by uid 500); 21 Oct 2008 18:12:59 -0000 Delivered-To: apmail-hadoop-core-commits-archive@hadoop.apache.org Received: (qmail 46297 invoked by uid 500); 21 Oct 2008 18:12:59 -0000 Mailing-List: contact core-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: core-dev@hadoop.apache.org Delivered-To: mailing list core-commits@hadoop.apache.org Received: (qmail 46270 invoked by uid 99); 21 Oct 2008 18:12:58 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 21 Oct 2008 11:12:58 -0700 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 21 Oct 2008 18:11:45 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 923882388AB0; Tue, 21 Oct 2008 11:11:53 -0700 (PDT) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r706704 [5/23] - in /hadoop/core/trunk: ./ src/contrib/hive/ src/contrib/hive/bin/ src/contrib/hive/cli/src/java/org/apache/hadoop/hive/cli/ src/contrib/hive/common/src/java/org/apache/hadoop/hive/conf/ src/contrib/hive/conf/ src/contrib/hi... Date: Tue, 21 Oct 2008 18:11:18 -0000 To: core-commits@hadoop.apache.org From: dhruba@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20081021181153.923882388AB0@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Modified: hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/exprNodeFieldDesc.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/exprNodeFieldDesc.java?rev=706704&r1=706703&r2=706704&view=diff ============================================================================== --- hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/exprNodeFieldDesc.java (original) +++ hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/exprNodeFieldDesc.java Tue Oct 21 11:11:05 2008 @@ -25,13 +25,18 @@ public class exprNodeFieldDesc extends exprNodeDesc implements Serializable { private static final long serialVersionUID = 1L; exprNodeDesc desc; - String fieldName; + String fieldName; + + // Used to support a.b where a is a list of struct that contains a field called b. + // a.b will return an array that contains field b of all elements of array a. + Boolean isList; public exprNodeFieldDesc() {} - public exprNodeFieldDesc(TypeInfo typeInfo, exprNodeDesc desc, String fieldName) { + public exprNodeFieldDesc(TypeInfo typeInfo, exprNodeDesc desc, String fieldName, Boolean isList) { super(typeInfo); this.desc = desc; - this.fieldName = fieldName; + this.fieldName = fieldName; + this.isList = isList; } public exprNodeDesc getDesc() { @@ -45,7 +50,14 @@ } public void setFieldName(String fieldName) { this.fieldName = fieldName; - } + } + public Boolean getIsList() { + return isList; + } + public void setIsList(Boolean isList) { + this.isList = isList; + } + @Override public String toString() { return this.desc.toString() + "." + this.fieldName; Added: hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/fetchWork.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/fetchWork.java?rev=706704&view=auto ============================================================================== --- hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/fetchWork.java (added) +++ hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/fetchWork.java Tue Oct 21 11:11:05 2008 @@ -0,0 +1,131 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.plan; + +import java.io.Serializable; +import java.util.Properties; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.serde2.Deserializer; +import org.apache.hadoop.mapred.InputFormat; + +@explain(displayName="Fetch Operator") +public class fetchWork implements Serializable { + private static final long serialVersionUID = 1L; + + // private loadFileDesc loadFileWork; + // private tableDesc tblDesc; + private Path srcDir; + private Properties schema; + private Class deserializerClass; + private Class inputFormatClass; + private int limit; + + public fetchWork() { } + + /** + * @param deserializer + * @param deserializerClass + * @param inputFormatClass + * @param schema + * @param srcDir + */ + public fetchWork(Path srcDir, + Class deserializerClass, + Class inputFormatClass, Properties schema, + int limit) { + this.srcDir = srcDir; + this.deserializerClass = deserializerClass; + this.inputFormatClass = inputFormatClass; + this.schema = schema; + this.limit = limit; + } + + /** + * @return the srcDir + */ + @explain(displayName="source") + public Path getSrcDir() { + return srcDir; + } + + /** + * @param srcDir the srcDir to set + */ + public void setSrcDir(Path srcDir) { + this.srcDir = srcDir; + } + + /** + * @return the schema + */ + public Properties getSchema() { + return schema; + } + + /** + * @param schema the schema to set + */ + public void setSchema(Properties schema) { + this.schema = schema; + } + + /** + * @return the deserializerClass + */ + public Class getDeserializerClass() { + return deserializerClass; + } + + /** + * @param deserializerClass the deserializerClass to set + */ + public void setDeserializerClass(Class deserializerClass) { + this.deserializerClass = deserializerClass; + } + + /** + * @return the inputFormatClass + */ + public Class getInputFormatClass() { + return inputFormatClass; + } + + /** + * @param inputFormatClass the inputFormatClass to set + */ + public void setInputFormatClass(Class inputFormatClass) { + this.inputFormatClass = inputFormatClass; + } + + /** + * @return the limit + */ + @explain(displayName="limit") + public int getLimit() { + return limit; + } + + /** + * @param limit the limit to set + */ + public void setLimit(int limit) { + this.limit = limit; + } +} Added: hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/limitDesc.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/limitDesc.java?rev=706704&view=auto ============================================================================== --- hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/limitDesc.java (added) +++ hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/limitDesc.java Tue Oct 21 11:11:05 2008 @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.plan; + +import java.io.Serializable; + +@explain(displayName="Limit") +public class limitDesc implements Serializable { + private static final long serialVersionUID = 1L; + private int limit; + public limitDesc() { } + public limitDesc(final int limit) { + this.limit = limit; + } + + public int getLimit() { + return this.limit; + } + public void setLimit(final int limit) { + this.limit=limit; + } + +} Modified: hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/loadFileDesc.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/loadFileDesc.java?rev=706704&r1=706703&r2=706704&view=diff ============================================================================== --- hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/loadFileDesc.java (original) +++ hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/loadFileDesc.java Tue Oct 21 11:11:05 2008 @@ -19,22 +19,28 @@ package org.apache.hadoop.hive.ql.plan; import java.io.Serializable; +import java.util.List; + import org.apache.hadoop.hive.ql.plan.loadDesc; public class loadFileDesc extends loadDesc implements Serializable { private static final long serialVersionUID = 1L; private String targetDir; private boolean isDfsDir; + // list of columns, comma separated + private String columns; public loadFileDesc() { } public loadFileDesc( final String sourceDir, final String targetDir, - final boolean isDfsDir) { + final boolean isDfsDir, + final String columns) { super(sourceDir); this.targetDir = targetDir; this.isDfsDir = isDfsDir; + this.columns = columns; } @explain(displayName="destination") @@ -52,4 +58,18 @@ public void setIsDfsDir(final boolean isDfsDir) { this.isDfsDir = isDfsDir; } + + /** + * @return the columns + */ + public String getColumns() { + return columns; + } + + /** + * @param columns the columns to set + */ + public void setColumns(String columns) { + this.columns = columns; + } } Modified: hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/mapredWork.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/mapredWork.java?rev=706704&r1=706703&r2=706704&view=diff ============================================================================== --- hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/mapredWork.java (original) +++ hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/mapredWork.java Tue Oct 21 11:11:05 2008 @@ -49,6 +49,7 @@ private Integer numReduceTasks; private boolean needsTagging; + private boolean inferNumReducers; public mapredWork() { } public mapredWork( @@ -197,4 +198,13 @@ public void setNeedsTagging(boolean needsTagging) { this.needsTagging = needsTagging; } + + public boolean getInferNumReducers() { + return this.inferNumReducers; + } + + public void setInferNumReducers(boolean inferNumReducers) { + this.inferNumReducers = inferNumReducers; + } + } Modified: hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/reduceSinkDesc.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/reduceSinkDesc.java?rev=706704&r1=706703&r2=706704&view=diff ============================================================================== --- hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/reduceSinkDesc.java (original) +++ hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/reduceSinkDesc.java Tue Oct 21 11:11:05 2008 @@ -38,33 +38,25 @@ // If the value is -1, then data will go to a random reducer private int numPartitionFields; - public reduceSinkDesc() { } + private boolean inferNumReducers; + private int numReducers; - public reduceSinkDesc - (final java.util.ArrayList keyCols, - final java.util.ArrayList valueCols, - final int numPartitionFields, - final tableDesc keySerializeInfo, - final tableDesc valueSerializeInfo) { - this.keyCols = keyCols; - this.valueCols = valueCols; - this.tag = -1; - this.numPartitionFields = numPartitionFields; - this.keySerializeInfo = keySerializeInfo; - this.valueSerializeInfo = valueSerializeInfo; - } + public reduceSinkDesc() { } public reduceSinkDesc (java.util.ArrayList keyCols, java.util.ArrayList valueCols, int tag, int numPartitionFields, + int numReducers, + boolean inferNumReducers, final tableDesc keySerializeInfo, final tableDesc valueSerializeInfo) { this.keyCols = keyCols; this.valueCols = valueCols; - assert tag != -1; this.tag = tag; + this.numReducers = numReducers; + this.inferNumReducers = inferNumReducers; this.numPartitionFields = numPartitionFields; this.keySerializeInfo = keySerializeInfo; this.valueSerializeInfo = valueSerializeInfo; @@ -104,6 +96,20 @@ this.tag = tag; } + public boolean getInferNumReducers() { + return this.inferNumReducers; + } + public void setInferNumReducers(boolean inferNumReducers) { + this.inferNumReducers = inferNumReducers; + } + + public int getNumReducers() { + return this.numReducers; + } + public void setNumReducers(int numReducers) { + this.numReducers = numReducers; + } + public tableDesc getKeySerializeInfo() { return keySerializeInfo; } Added: hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/showPartitionsDesc.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/showPartitionsDesc.java?rev=706704&view=auto ============================================================================== --- hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/showPartitionsDesc.java (added) +++ hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/showPartitionsDesc.java Tue Oct 21 11:11:05 2008 @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.plan; + +import java.io.Serializable; +import org.apache.hadoop.fs.Path; + +@explain(displayName="Show Partitions") +public class showPartitionsDesc extends ddlDesc implements Serializable +{ + private static final long serialVersionUID = 1L; + String tabName; + Path resFile; + + /** + * @param tabName Name of the table whose partitions need to be listed + * @param resFile File to store the results in + */ + public showPartitionsDesc(String tabName, Path resFile) { + this.tabName = tabName; + this.resFile = resFile; + } + + /** + * @return the name of the table + */ + @explain(displayName="table") + public String getTabName() { + return tabName; + } + + /** + * @param tabName the table whose partitions have to be listed + */ + public void setTabName(String tabName) { + this.tabName = tabName; + } + + /** + * @return the results file + */ + public Path getResFile() { + return resFile; + } + + @explain(displayName="result file", normalExplain=false) + public String getResFileString() { + return getResFile().getName(); + } + /** + * @param resFile the results file to be used to return the results + */ + public void setResFile(Path resFile) { + this.resFile = resFile; + } +} Modified: hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/tableDesc.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/tableDesc.java?rev=706704&r1=706703&r2=706704&view=diff ============================================================================== --- hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/tableDesc.java (original) +++ hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/tableDesc.java Tue Oct 21 11:11:05 2008 @@ -47,7 +47,7 @@ public Class getDeserializerClass() { return this.deserializerClass; } - public void setDeserializerClass(final Class serdeClass) { + public void setDeserializerClass(final Class serdeClass) { this.deserializerClass = serdeClass; } public Class getInputFileFormatClass() { Modified: hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/typeinfo/StructTypeInfo.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/typeinfo/StructTypeInfo.java?rev=706704&r1=706703&r2=706704&view=diff ============================================================================== --- hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/typeinfo/StructTypeInfo.java (original) +++ hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/typeinfo/StructTypeInfo.java Tue Oct 21 11:11:05 2008 @@ -95,12 +95,14 @@ } public TypeInfo getStructFieldTypeInfo(String field) { + String fieldLowerCase = field.toLowerCase(); for(int i=0; i s) { + if (s == null) { + return -1; + } + return s.size(); + } + + public Integer evaluate(List s) { + if (s == null) { + return -1; + } + return s.size(); + } + +} Modified: hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFStrEq.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFStrEq.java?rev=706704&r1=706703&r2=706704&view=diff ============================================================================== --- hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFStrEq.java (original) +++ hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFStrEq.java Tue Oct 21 11:11:05 2008 @@ -23,7 +23,7 @@ import org.apache.hadoop.hive.ql.exec.UDF; -public class UDFStrEq extends UDF { +public class UDFStrEq implements UDF { private static Log LOG = LogFactory.getLog("org.apache.hadoop.hive.ql.udf.UDFStrEq"); Modified: hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFStrGe.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFStrGe.java?rev=706704&r1=706703&r2=706704&view=diff ============================================================================== --- hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFStrGe.java (original) +++ hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFStrGe.java Tue Oct 21 11:11:05 2008 @@ -23,7 +23,7 @@ import org.apache.hadoop.hive.ql.exec.UDF; -public class UDFStrGe extends UDF { +public class UDFStrGe implements UDF { public UDFStrGe() { } Modified: hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFStrGt.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFStrGt.java?rev=706704&r1=706703&r2=706704&view=diff ============================================================================== --- hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFStrGt.java (original) +++ hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFStrGt.java Tue Oct 21 11:11:05 2008 @@ -23,7 +23,7 @@ import org.apache.hadoop.hive.ql.exec.UDF; -public class UDFStrGt extends UDF { +public class UDFStrGt implements UDF { public UDFStrGt() { } Modified: hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFStrLe.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFStrLe.java?rev=706704&r1=706703&r2=706704&view=diff ============================================================================== --- hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFStrLe.java (original) +++ hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFStrLe.java Tue Oct 21 11:11:05 2008 @@ -23,7 +23,7 @@ import org.apache.hadoop.hive.ql.exec.UDF; -public class UDFStrLe extends UDF { +public class UDFStrLe implements UDF { public UDFStrLe() { } Modified: hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFStrLt.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFStrLt.java?rev=706704&r1=706703&r2=706704&view=diff ============================================================================== --- hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFStrLt.java (original) +++ hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFStrLt.java Tue Oct 21 11:11:05 2008 @@ -23,7 +23,7 @@ import org.apache.hadoop.hive.ql.exec.UDF; -public class UDFStrLt extends UDF { +public class UDFStrLt implements UDF { public UDFStrLt() { } Modified: hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFStrNe.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFStrNe.java?rev=706704&r1=706703&r2=706704&view=diff ============================================================================== --- hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFStrNe.java (original) +++ hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFStrNe.java Tue Oct 21 11:11:05 2008 @@ -23,7 +23,7 @@ import org.apache.hadoop.hive.ql.exec.UDF; -public class UDFStrNe extends UDF { +public class UDFStrNe implements UDF { public UDFStrNe() { } Modified: hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFSubstr.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFSubstr.java?rev=706704&r1=706703&r2=706704&view=diff ============================================================================== --- hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFSubstr.java (original) +++ hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFSubstr.java Tue Oct 21 11:11:05 2008 @@ -21,7 +21,7 @@ import org.apache.hadoop.hive.ql.exec.UDF; -public class UDFSubstr extends UDF { +public class UDFSubstr implements UDF { public UDFSubstr() { } Modified: hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToBoolean.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToBoolean.java?rev=706704&r1=706703&r2=706704&view=diff ============================================================================== --- hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToBoolean.java (original) +++ hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToBoolean.java Tue Oct 21 11:11:05 2008 @@ -23,7 +23,7 @@ import org.apache.hadoop.hive.ql.exec.UDF; -public class UDFToBoolean extends UDF { +public class UDFToBoolean implements UDF { private static Log LOG = LogFactory.getLog(UDFToBoolean.class.getName()); Modified: hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToByte.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToByte.java?rev=706704&r1=706703&r2=706704&view=diff ============================================================================== --- hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToByte.java (original) +++ hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToByte.java Tue Oct 21 11:11:05 2008 @@ -23,7 +23,7 @@ import org.apache.hadoop.hive.ql.exec.UDF; -public class UDFToByte extends UDF { +public class UDFToByte implements UDF { private static Log LOG = LogFactory.getLog(UDFToByte.class.getName()); Modified: hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToDate.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToDate.java?rev=706704&r1=706703&r2=706704&view=diff ============================================================================== --- hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToDate.java (original) +++ hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToDate.java Tue Oct 21 11:11:05 2008 @@ -23,7 +23,7 @@ import org.apache.hadoop.hive.ql.exec.UDF; -public class UDFToDate extends UDF { +public class UDFToDate implements UDF { private static Log LOG = LogFactory.getLog(UDFToDate.class.getName()); Modified: hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToDouble.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToDouble.java?rev=706704&r1=706703&r2=706704&view=diff ============================================================================== --- hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToDouble.java (original) +++ hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToDouble.java Tue Oct 21 11:11:05 2008 @@ -23,7 +23,7 @@ import org.apache.hadoop.hive.ql.exec.UDF; -public class UDFToDouble extends UDF { +public class UDFToDouble implements UDF { private static Log LOG = LogFactory.getLog(UDFToDouble.class.getName()); Modified: hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToFloat.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToFloat.java?rev=706704&r1=706703&r2=706704&view=diff ============================================================================== --- hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToFloat.java (original) +++ hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToFloat.java Tue Oct 21 11:11:05 2008 @@ -23,7 +23,7 @@ import org.apache.hadoop.hive.ql.exec.UDF; -public class UDFToFloat extends UDF { +public class UDFToFloat implements UDF { private static Log LOG = LogFactory.getLog(UDFToFloat.class.getName()); Modified: hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToInteger.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToInteger.java?rev=706704&r1=706703&r2=706704&view=diff ============================================================================== --- hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToInteger.java (original) +++ hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToInteger.java Tue Oct 21 11:11:05 2008 @@ -23,7 +23,7 @@ import org.apache.hadoop.hive.ql.exec.UDF; -public class UDFToInteger extends UDF { +public class UDFToInteger implements UDF { private static Log LOG = LogFactory.getLog(UDFToInteger.class.getName()); Modified: hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToLong.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToLong.java?rev=706704&r1=706703&r2=706704&view=diff ============================================================================== --- hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToLong.java (original) +++ hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToLong.java Tue Oct 21 11:11:05 2008 @@ -23,7 +23,7 @@ import org.apache.hadoop.hive.ql.exec.UDF; -public class UDFToLong extends UDF { +public class UDFToLong implements UDF { private static Log LOG = LogFactory.getLog(UDFToLong.class.getName()); Modified: hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToString.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToString.java?rev=706704&r1=706703&r2=706704&view=diff ============================================================================== --- hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToString.java (original) +++ hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToString.java Tue Oct 21 11:11:05 2008 @@ -23,7 +23,7 @@ import org.apache.hadoop.hive.ql.exec.UDF; -public class UDFToString extends UDF { +public class UDFToString implements UDF { private static Log LOG = LogFactory.getLog(UDFToString.class.getName()); Modified: hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFTrim.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFTrim.java?rev=706704&r1=706703&r2=706704&view=diff ============================================================================== --- hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFTrim.java (original) +++ hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFTrim.java Tue Oct 21 11:11:05 2008 @@ -24,7 +24,7 @@ import java.util.regex.Pattern; import java.util.regex.Matcher; -public class UDFTrim extends UDF { +public class UDFTrim implements UDF { public UDFTrim() { } @@ -33,7 +33,7 @@ if (s == null) { return null; } - return StringUtils.stripEnd(s, " "); + return StringUtils.strip(s, " "); } } Modified: hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFUpper.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFUpper.java?rev=706704&r1=706703&r2=706704&view=diff ============================================================================== --- hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFUpper.java (original) +++ hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFUpper.java Tue Oct 21 11:11:05 2008 @@ -22,7 +22,7 @@ import java.util.regex.Pattern; import java.util.regex.Matcher; -public class UDFUpper extends UDF { +public class UDFUpper implements UDF { public UDFUpper() { } Modified: hadoop/core/trunk/src/contrib/hive/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java?rev=706704&r1=706703&r2=706704&view=diff ============================================================================== --- hadoop/core/trunk/src/contrib/hive/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java (original) +++ hadoop/core/trunk/src/contrib/hive/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java Tue Oct 21 11:11:05 2008 @@ -53,7 +53,7 @@ import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.session.SessionState; -import org.apache.hadoop.hive.ql.thrift.Complex; +import org.apache.hadoop.hive.serde2.thrift.test.Complex; import org.apache.hadoop.hive.serde.Constants; import org.apache.hadoop.mapred.SequenceFileInputFormat; import org.apache.hadoop.mapred.SequenceFileOutputFormat; @@ -277,7 +277,7 @@ Table srcThrift = new Table("src_thrift"); srcThrift.setInputFormatClass(SequenceFileInputFormat.class.getName()); srcThrift.setOutputFormatClass(SequenceFileOutputFormat.class.getName()); - srcThrift.setSerializationLib(ThriftDeserializer.shortName()); + srcThrift.setSerializationLib(ThriftDeserializer.class.getName()); srcThrift.setSerdeParam(Constants.SERIALIZATION_CLASS, Complex.class.getName()); srcThrift.setSerdeParam(Constants.SERIALIZATION_FORMAT, TBinaryProtocol.class.getName()); db.createTable(srcThrift); @@ -364,7 +364,6 @@ CliSessionState ss = new CliSessionState(new HiveConf(SessionState.class)); ss.in = System.in; - ss.err = System.err; File qf = new File(outDir, tname); File outf = null; @@ -372,6 +371,7 @@ outf = new File(outf, qf.getName().concat(".out")); FileOutputStream fo = new FileOutputStream(outf); ss.out = new PrintStream(fo, true, "UTF-8"); + ss.err = ss.out; ss.setIsSilent(true); cliDriver = new CliDriver(ss); SessionState.start(ss); @@ -644,7 +644,10 @@ public List> analyzeAST(CommonTree ast) throws Exception { // Do semantic analysis and plan generation - sem.analyze(ast, new Context(conf)); + Context ctx = new Context(conf); + ctx.makeScratchDir(); + sem.analyze(ast, ctx); + ctx.removeScratchDir(); return sem.getRootTasks(); } Modified: hadoop/core/trunk/src/contrib/hive/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java?rev=706704&r1=706703&r2=706704&view=diff ============================================================================== --- hadoop/core/trunk/src/contrib/hive/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java (original) +++ hadoop/core/trunk/src/contrib/hive/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java Tue Oct 21 11:11:05 2008 @@ -203,8 +203,7 @@ Operator op1 = OperatorFactory.get (PlanUtils.getReduceSinkDesc (Utilities.makeList(new exprNodeColumnDesc(String.class, "key")), - Utilities.makeList(new exprNodeColumnDesc(String.class, "value")), - 1)); + Utilities.makeList(new exprNodeColumnDesc(String.class, "value")), -1, 1, -1, false)); Utilities.addMapWork(mr, src, "a", op1); @@ -228,8 +227,7 @@ (PlanUtils.getReduceSinkDesc (Utilities.makeList(new exprNodeColumnDesc(String.class, "key")), Utilities.makeList(new exprNodeColumnDesc(String.class, "key"), - new exprNodeColumnDesc(String.class, "value")), - 1)); + new exprNodeColumnDesc(String.class, "value")), -1, 1, -1, false)); Utilities.addMapWork(mr, src, "a", op1); @@ -260,8 +258,7 @@ (PlanUtils.getReduceSinkDesc (Utilities.makeList(new exprNodeColumnDesc(String.class, "key")), Utilities.makeList - (new exprNodeColumnDesc(String.class, "value")), Byte.valueOf((byte)0), - 1)); + (new exprNodeColumnDesc(String.class, "value")), Byte.valueOf((byte)0), 1, -1, false)); Utilities.addMapWork(mr, src, "a", op1); @@ -270,7 +267,7 @@ (Utilities.makeList(new exprNodeColumnDesc(String.class, "key")), Utilities.makeList(new exprNodeColumnDesc(String.class, "key")), Byte.valueOf((byte)1), - Integer.MAX_VALUE)); + Integer.MAX_VALUE, -1, false)); Utilities.addMapWork(mr, src2, "b", op2); @@ -291,7 +288,8 @@ new exprNodeColumnDesc(TypeInfoFactory.getListTypeInfo( TypeInfoFactory.getPrimitiveTypeInfo(String.class)), Utilities.ReduceField.VALUE.toString()), - "0"))), op4); + "0", + false))), op4); mr.setReducer(op5); } @@ -307,7 +305,7 @@ (Utilities.makeList(new exprNodeColumnDesc(String.class, "tkey")), Utilities.makeList(new exprNodeColumnDesc(String.class, "tkey"), new exprNodeColumnDesc(String.class, "tvalue")), - 1)); + -1, 1, -1, false)); Operator op0 = OperatorFactory.get (new scriptDesc("/bin/cat", @@ -343,7 +341,7 @@ (Utilities.makeList(new exprNodeColumnDesc(String.class, "0")), Utilities.makeList(new exprNodeColumnDesc(String.class, "0"), new exprNodeColumnDesc(String.class, "1")), - 1)); + -1, 1, -1, false)); Operator op4 = OperatorFactory.get(new selectDesc( Utilities.makeList(new exprNodeColumnDesc(String.class, "key"), @@ -373,7 +371,7 @@ Utilities.makeList(new exprNodeColumnDesc(String.class, "tkey")), Utilities.makeList(new exprNodeColumnDesc(String.class, "tkey"), new exprNodeColumnDesc(String.class, "tvalue")), - 1)); + -1, 1, -1, false)); Operator op0 = OperatorFactory.get (new scriptDesc("\'/bin/cat\'", Modified: hadoop/core/trunk/src/contrib/hive/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java?rev=706704&r1=706703&r2=706704&view=diff ============================================================================== --- hadoop/core/trunk/src/contrib/hive/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java (original) +++ hadoop/core/trunk/src/contrib/hive/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java Tue Oct 21 11:11:05 2008 @@ -29,7 +29,7 @@ import org.apache.hadoop.hive.metastore.DB; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat; -import org.apache.hadoop.hive.ql.thrift.Complex; +import org.apache.hadoop.hive.serde2.thrift.test.Complex; import org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe; import org.apache.hadoop.hive.serde2.ThriftDeserializer; import org.apache.hadoop.hive.serde.Constants; @@ -137,7 +137,8 @@ // now that URI is set correctly, set the original table's uri and then compare the two tables tbl.setDataLocation(ft.getDataLocation()); assertTrue("Tables doesn't match: " + tableName, ft.getTTable().equals(tbl.getTTable())); - assertEquals("Serde is not set correctly", tbl.getDeserializer().getShortName(), ft.getDeserializer().getShortName()); + assertEquals("Serde is not set correctly", tbl.getDeserializer().getClass().getName(), ft.getDeserializer().getClass().getName()); + assertEquals("SerializationLib is not set correctly", tbl.getSerializationLib(), MetadataTypedColumnsetSerDe.class.getName()); } catch (HiveException e) { e.printStackTrace(); assertTrue("Unable to fetch table correctly: " + tableName, false); @@ -195,7 +196,8 @@ // now that URI is set correctly, set the original table's uri and then compare the two tables tbl.setDataLocation(ft.getDataLocation()); assertTrue("Tables doesn't match: " + tableName, ft.getTTable().equals(tbl.getTTable())); - assertEquals("Serde is not set correctly", tbl.getDeserializer().getShortName(), ft.getDeserializer().getShortName()); + assertEquals("SerializationLib is not set correctly", tbl.getSerializationLib(), ThriftDeserializer.class.getName()); + assertEquals("Serde is not set correctly", tbl.getDeserializer().getClass().getName(), ft.getDeserializer().getClass().getName()); } catch (HiveException e) { System.err.println(StringUtils.stringifyException(e)); assertTrue("Unable to fetch table correctly: " + tableName, false); Modified: hadoop/core/trunk/src/contrib/hive/ql/src/test/org/apache/hadoop/hive/ql/udf/UDFTestLength.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/ql/src/test/org/apache/hadoop/hive/ql/udf/UDFTestLength.java?rev=706704&r1=706703&r2=706704&view=diff ============================================================================== --- hadoop/core/trunk/src/contrib/hive/ql/src/test/org/apache/hadoop/hive/ql/udf/UDFTestLength.java (original) +++ hadoop/core/trunk/src/contrib/hive/ql/src/test/org/apache/hadoop/hive/ql/udf/UDFTestLength.java Tue Oct 21 11:11:05 2008 @@ -23,7 +23,7 @@ /** * A UDF for testing, which evaluates the length of a string. */ -public class UDFTestLength extends UDF { +public class UDFTestLength implements UDF { public Integer evaluate(String s) { return s == null ? null : s.length(); }