incubator-hcatalog-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1197380 - in /incubator/hcatalog/trunk: ./ hive/ src/java/org/apache/hcatalog/cli/ src/java/org/apache/hcatalog/data/schema/ src/java/org/apache/hcatalog/pig/ src/test/org/apache/hcatalog/cli/ src/test/org/apache/hcatalog/pig/
Date Fri, 04 Nov 2011 01:22:36 GMT
Author: hashutosh
Date: Fri Nov  4 01:22:36 2011
New Revision: 1197380

URL: http://svn.apache.org/viewvc?rev=1197380&view=rev
Log:
Add support for binary data type

Modified:
    incubator/hcatalog/trunk/CHANGES.txt
    incubator/hcatalog/trunk/build-common.xml
    incubator/hcatalog/trunk/build.xml
    incubator/hcatalog/trunk/hive/   (props changed)
    incubator/hcatalog/trunk/src/java/org/apache/hcatalog/cli/HCatCli.java
    incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatFieldSchema.java
    incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatSchemaUtils.java
    incubator/hcatalog/trunk/src/java/org/apache/hcatalog/pig/HCatBaseStorer.java
    incubator/hcatalog/trunk/src/java/org/apache/hcatalog/pig/PigHCatUtil.java
    incubator/hcatalog/trunk/src/test/org/apache/hcatalog/cli/TestSemanticAnalysis.java
    incubator/hcatalog/trunk/src/test/org/apache/hcatalog/pig/TestHCatStorer.java

Modified: incubator/hcatalog/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/CHANGES.txt?rev=1197380&r1=1197379&r2=1197380&view=diff
==============================================================================
--- incubator/hcatalog/trunk/CHANGES.txt (original)
+++ incubator/hcatalog/trunk/CHANGES.txt Fri Nov  4 01:22:36 2011
@@ -23,6 +23,8 @@ Trunk (unreleased changes)
   INCOMPATIBLE CHANGES
 
   NEW FEATURES
+  HCAT-145. Add support for binary data type (hashutosh)  
+
   HCAT-151. Fixed native table names used for tables stored in non-default DBs in HBaseInputStorageDriver
(avandana via toffer)
 
   HCAT-143. Projection pushdown for HBaseInputStorageDriver (avandana via toffer)

Modified: incubator/hcatalog/trunk/build-common.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/build-common.xml?rev=1197380&r1=1197379&r2=1197380&view=diff
==============================================================================
--- incubator/hcatalog/trunk/build-common.xml (original)
+++ incubator/hcatalog/trunk/build-common.xml Fri Nov  4 01:22:36 2011
@@ -15,7 +15,6 @@
     <!-- common classpaths for various builds -->
     <path id="common.classpath">
         <fileset dir="${hive.root}" includes="testlibs/*.jar"/>
-        <fileset dir="${hive.root}" includes="testlibs/*.jar"/>
         <fileset dir="${hive.root}/lib" includes="*.jar"/>
         <fileset dir="${hive.root}/build/cli" includes="*.jar"/>
         <fileset dir="${hive.root}/build/common" includes="*.jar"/>
@@ -29,6 +28,7 @@
         <fileset dir="${hive.root}/build/ivy/lib/default" includes="jdo2-api-2.3-ec.jar"/>
         <fileset dir="${hive.root}/build/ivy/lib/default" includes="datanucleus-enhancer-2.0.3.jar"/>
         <fileset dir="${hive.root}/build/ivy/lib/default" includes="datanucleus-core-2.0.3.jar"/>
+        <fileset dir="${hive.root}/build/ivy/lib/default" includes="libfb303-*.jar"/>
         <fileset dir="${common.lib.dir}" includes="hadoop_archive-0.3.1.jar"/>
         <fileset dir="${hive.root}/lib" includes="asm-3.1.jar"/>
     </path>

Modified: incubator/hcatalog/trunk/build.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/build.xml?rev=1197380&r1=1197379&r2=1197380&view=diff
==============================================================================
--- incubator/hcatalog/trunk/build.xml (original)
+++ incubator/hcatalog/trunk/build.xml Fri Nov  4 01:22:36 2011
@@ -451,6 +451,7 @@
         <include name="commons-pool-*.jar"/>
         <include name="commons-dbcp-*.jar"/>
         <include name="guava-*.jar"/>
+        <include name="libthrift-*.jar"/>
       </fileset>
 
       <fileset dir="hive/external/build/shims">
@@ -488,7 +489,6 @@
         <include name="commons-lang-*.jar"/>
         <include name="asm-api-*.jar"/>
         <include name="derby-*.jar"/>
-        <include name="thrift-fb303-*.jar"/>
       </fileset>
 
       <fileset dir="hive/external/cli/lib">

Propchange: incubator/hcatalog/trunk/hive/
------------------------------------------------------------------------------
--- svn:externals (original)
+++ svn:externals Fri Nov  4 01:22:36 2011
@@ -1 +1 @@
--r 1148878 http://svn.apache.org/repos/asf/hive/trunk external
+http://svn.apache.org/repos/asf/hive/trunk external

Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/cli/HCatCli.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/cli/HCatCli.java?rev=1197380&r1=1197379&r2=1197380&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/cli/HCatCli.java (original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/cli/HCatCli.java Fri Nov  4 01:22:36
2011
@@ -38,10 +38,11 @@ import org.apache.commons.cli.Parser;
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hive.cli.CliSessionState;
+import org.apache.hadoop.hive.common.LogUtils;
+import org.apache.hadoop.hive.common.LogUtils.LogInitializationException;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.hive.ql.CommandNeedRetryException;
-import org.apache.hadoop.hive.ql.Driver;
 import org.apache.hadoop.hive.ql.processors.SetProcessor;
 import org.apache.hadoop.hive.ql.session.SessionState;
 import org.apache.hcatalog.cli.SemanticAnalysis.HCatSemanticAnalyzer;
@@ -53,7 +54,11 @@ public class HCatCli {
   @SuppressWarnings("static-access")
   public static void main(String[] args) {
 
-    SessionState.initHiveLog4j();
+	  try {
+		  LogUtils.initHiveLog4j();
+	  } catch (LogInitializationException e) {
+
+	  }
 
     CliSessionState ss = new CliSessionState(new HiveConf(SessionState.class));
     ss.in = System.in;

Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatFieldSchema.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatFieldSchema.java?rev=1197380&r1=1197379&r2=1197380&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatFieldSchema.java
(original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatFieldSchema.java
Fri Nov  4 01:22:36 2011
@@ -35,6 +35,7 @@ public class HCatFieldSchema implements 
         ARRAY,
         MAP,
         STRUCT,
+        BINARY,
     }
 
     public enum Category {

Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatSchemaUtils.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatSchemaUtils.java?rev=1197380&r1=1197379&r2=1197380&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatSchemaUtils.java
(original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatSchemaUtils.java
Fri Nov  4 01:22:36 2011
@@ -151,6 +151,8 @@ public class HCatSchemaUtils {
             return Type.SMALLINT;
         case STRING:
             return Type.STRING;
+        case BINARY:
+        	return Type.BINARY;
         default:
             throw new TypeNotPresentException(((PrimitiveTypeInfo)basePrimitiveTypeInfo).getTypeName(),
null);
         }

Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/pig/HCatBaseStorer.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/pig/HCatBaseStorer.java?rev=1197380&r1=1197379&r2=1197380&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/pig/HCatBaseStorer.java (original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/pig/HCatBaseStorer.java Fri Nov
 4 01:22:36 2011
@@ -26,6 +26,7 @@ import java.util.List;
 import java.util.Map;
 
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef;
 import org.apache.hadoop.io.WritableComparable;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.RecordWriter;
@@ -42,6 +43,7 @@ import org.apache.pig.StoreMetadata;
 import org.apache.pig.backend.BackendException;
 import org.apache.pig.backend.executionengine.ExecException;
 import org.apache.pig.data.DataBag;
+import org.apache.pig.data.DataByteArray;
 import org.apache.pig.data.DataType;
 import org.apache.pig.data.Tuple;
 import org.apache.pig.impl.logicalLayer.FrontendException;
@@ -187,6 +189,9 @@ public abstract class HCatBaseStorer ext
     case DataType.DOUBLE:
       return new HCatFieldSchema(fSchema.alias, Type.DOUBLE, null);
 
+    case DataType.BYTEARRAY:
+    	return new HCatFieldSchema(fSchema.alias, Type.BINARY, null);
+    	
     case DataType.BAG:
       Schema bagSchema = fSchema.schema;
       List<HCatFieldSchema> arrFields = new ArrayList<HCatFieldSchema>(1);
@@ -219,6 +224,7 @@ public abstract class HCatBaseStorer ext
         case INT:
         case FLOAT:
         case DOUBLE:
+        case BINARY:
           valFS = new HCatFieldSchema(fSchema.alias, mapValType, null);
           break;
         default:
@@ -269,6 +275,12 @@ public abstract class HCatBaseStorer ext
 
     switch(type){
 
+    case BINARY:
+    	ByteArrayRef ba = new ByteArrayRef();
+    	byte[] bytes = (null == pigObj) ? new byte[0] : ((DataByteArray)pigObj).get(); 
+    	ba.setData(bytes);
+    	return ba;
+    	
     case STRUCT:
       // Unwrap the tuple.
       return ((Tuple)pigObj).getAll();

Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/pig/PigHCatUtil.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/pig/PigHCatUtil.java?rev=1197380&r1=1197379&r2=1197380&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/pig/PigHCatUtil.java (original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/pig/PigHCatUtil.java Fri Nov  4
01:22:36 2011
@@ -31,6 +31,8 @@ import org.apache.hadoop.hive.metastore.
 import org.apache.hadoop.hive.metastore.MetaStoreUtils;
 import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
 import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef;
+import org.apache.hadoop.io.BytesWritable;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hcatalog.common.HCatConstants;
 import org.apache.hcatalog.data.HCatArrayBag;
@@ -44,6 +46,7 @@ import org.apache.pig.ResourceSchema;
 import org.apache.pig.LoadPushDown.RequiredField;
 import org.apache.pig.ResourceSchema.ResourceFieldSchema;
 import org.apache.pig.data.DataBag;
+import org.apache.pig.data.DataByteArray;
 import org.apache.pig.data.DataType;
 import org.apache.pig.data.DefaultDataBag;
 import org.apache.pig.data.Tuple;
@@ -275,6 +278,10 @@ public class PigHCatUtil {
       return DataType.DOUBLE;
     }
 
+    if (type == Type.BINARY){
+        return DataType.BYTEARRAY;
+    }
+    
     if (type == Type.BOOLEAN){
       errMsg = "HCatalog column type 'BOOLEAN' is not supported in " +
       "Pig as a column type";
@@ -293,18 +300,20 @@ public class PigHCatUtil {
     }
 
   @SuppressWarnings("unchecked")
-public static Object extractPigObject(Object o, HCatFieldSchema hfs) throws Exception {
+  public static Object extractPigObject(Object o, HCatFieldSchema hfs) throws Exception {
       Type itemType = hfs.getType();
-      if ( ! hfs.isComplex()){
-        return o;
-      } else  if (itemType == Type.STRUCT) {
-        return transformToTuple((List<Object>)o,hfs);
-      } else  if (itemType == Type.ARRAY) {
-        return transformToBag((List<? extends Object>) o,hfs);
-      } else  if (itemType == Type.MAP) {
-        return transformToPigMap((Map<String, Object>)o,hfs);
+      switch (itemType){
+	case BINARY:
+	  return new DataByteArray(((ByteArrayRef)o).getData());
+	case STRUCT:
+          return transformToTuple((List<Object>)o,hfs);
+	case ARRAY:
+          return transformToBag((List<? extends Object>) o,hfs);
+	case MAP:
+          return transformToPigMap((Map<String, Object>)o,hfs);
+	default:
+	  return o;
       }
-      return null; // never invoked.
   }
 
   public static Tuple transformToTuple(List<? extends Object> objList, HCatFieldSchema
hfs) throws Exception {

Modified: incubator/hcatalog/trunk/src/test/org/apache/hcatalog/cli/TestSemanticAnalysis.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/org/apache/hcatalog/cli/TestSemanticAnalysis.java?rev=1197380&r1=1197379&r2=1197380&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/org/apache/hcatalog/cli/TestSemanticAnalysis.java (original)
+++ incubator/hcatalog/trunk/src/test/org/apache/hcatalog/cli/TestSemanticAnalysis.java Fri
Nov  4 01:22:36 2011
@@ -218,8 +218,8 @@ public class TestSemanticAnalysis extend
     Table tbl = msc.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, tblName);
     List<FieldSchema> cols = tbl.getSd().getCols();
     assertEquals(3, cols.size());
-    assertTrue(cols.get(0).equals(new FieldSchema("a", "int", "from deserializer")));
-    assertTrue(cols.get(1).equals(new FieldSchema("c", "string", "from deserializer")));
+    assertTrue(cols.get(0).equals(new FieldSchema("a", "int", null)));
+    assertTrue(cols.get(1).equals(new FieldSchema("c", "string", null)));
     assertTrue(cols.get(2).equals(new FieldSchema("d", "tinyint", null)));
     hcatDriver.run("drop table junit_sem_analysis");
   }

Modified: incubator/hcatalog/trunk/src/test/org/apache/hcatalog/pig/TestHCatStorer.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/org/apache/hcatalog/pig/TestHCatStorer.java?rev=1197380&r1=1197379&r2=1197380&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/org/apache/hcatalog/pig/TestHCatStorer.java (original)
+++ incubator/hcatalog/trunk/src/test/org/apache/hcatalog/pig/TestHCatStorer.java Fri Nov
 4 01:22:36 2011
@@ -37,6 +37,7 @@ import org.apache.hcatalog.rcfile.RCFile
 import org.apache.pig.ExecType;
 import org.apache.pig.PigException;
 import org.apache.pig.PigServer;
+import org.apache.pig.data.DataByteArray;
 import org.apache.pig.data.Tuple;
 import org.apache.pig.impl.logicalLayer.FrontendException;
 import org.apache.pig.impl.util.LogUtils;
@@ -498,7 +499,7 @@ public class TestHCatStorer extends Test
   public void testStoreFuncAllSimpleTypes() throws IOException, CommandNeedRetryException{
 
     driver.run("drop table junit_unparted");
-    String createTable = "create table junit_unparted(a int, b float, c double, d bigint,
e string) stored as RCFILE " +
+    String createTable = "create table junit_unparted(a int, b float, c double, d bigint,
e string, f binary) stored as RCFILE " +
         "tblproperties('"+HCatConstants.HCAT_ISD_CLASS+"'='"+RCFileInputDriver.class.getName()+"',"
+
         "'"+HCatConstants.HCAT_OSD_CLASS+"'='"+RCFileOutputDriver.class.getName()+"') ";
     int retCode = driver.run(createTable).getResponseCode();
@@ -509,17 +510,17 @@ public class TestHCatStorer extends Test
     int LOOP_SIZE = 3;
     String[] input = new String[LOOP_SIZE*LOOP_SIZE];
     for(int i = 0; i < LOOP_SIZE*LOOP_SIZE; i++) {
-      input[i] = i + "\t" + i * 2.1f +"\t"+ i*1.1d + "\t" + i * 2L +"\t"+"lets hcat";
+      input[i] = i + "\t" + i * 2.1f +"\t"+ i*1.1d + "\t" + i * 2L +"\t"+"lets hcat"+"\tbinary-data";
     }
 
     MiniCluster.createInputFile(cluster, fileName, input);
     PigServer server = new PigServer(ExecType.LOCAL, props);
     UDFContext.getUDFContext().setClientSystemProps();
     server.setBatchOn();
-    server.registerQuery("A = load '"+fullFileName+"' as (a:int, b:float, c:double, d:long,
e:chararray);");
-    server.registerQuery("store A into 'default.junit_unparted' using "+HCatStorer.class.getName()+"('','a:int,
b:float, c:double, d:long, e:chararray');");
+    server.registerQuery("A = load '"+fullFileName+"' as (a:int, b:float, c:double, d:long,
e:chararray, f:bytearray);");
+    server.registerQuery("store A into 'default.junit_unparted' using "+HCatStorer.class.getName()+"('','a:int,
b:float, c:double, d:long, e:chararray,f:bytearray');");
     server.executeBatch();
-    MiniCluster.deleteFile(cluster, fileName);
+
 
     driver.run("select * from junit_unparted");
     ArrayList<String> res = new ArrayList<String>();
@@ -531,8 +532,19 @@ public class TestHCatStorer extends Test
     }
 
     assertFalse(itr.hasNext());
+    server.registerQuery("B = load 'junit_unparted' using "+HCatLoader.class.getName()+";");
+    Iterator<Tuple> i = server.openIterator("B");
+    int count = 0;
+    while(i.hasNext()){
+    	Object o = i.next().get(5);
+    	assertTrue(o instanceof DataByteArray);
+    	count++;
+    }
+    assertEquals(LOOP_SIZE  * LOOP_SIZE, count);
+    MiniCluster.deleteFile(cluster, fileName);
     driver.run("drop table junit_unparted");
   }
+  
   @Override
   protected void tearDown() throws Exception {
     super.tearDown();



Mime
View raw message