incubator-hcatalog-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tof...@apache.org
Subject svn commit: r1244334 [4/4] - in /incubator/hcatalog/trunk: ./ src/java/org/apache/hadoop/ src/java/org/apache/hadoop/mapred/ src/java/org/apache/hcatalog/cli/SemanticAnalysis/ src/java/org/apache/hcatalog/common/ src/java/org/apache/hcatalog/mapred/ sr...
Date Wed, 15 Feb 2012 03:53:52 GMT
Added: incubator/hcatalog/trunk/src/test/org/apache/hcatalog/pig/TestPigStorageDriver.java.broken
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/org/apache/hcatalog/pig/TestPigStorageDriver.java.broken?rev=1244334&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/test/org/apache/hcatalog/pig/TestPigStorageDriver.java.broken
(added)
+++ incubator/hcatalog/trunk/src/test/org/apache/hcatalog/pig/TestPigStorageDriver.java.broken
Wed Feb 15 03:53:50 2012
@@ -0,0 +1,270 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.pig;
+
+import java.io.BufferedInputStream;
+import java.io.DataInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.Iterator;
+import java.util.Map;
+
+import junit.framework.TestCase;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.cli.CliSessionState;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.MetaStoreUtils;
+import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.UnknownTableException;
+import org.apache.hadoop.hive.ql.CommandNeedRetryException;
+import org.apache.hadoop.hive.ql.Driver;
+import org.apache.hadoop.hive.ql.io.RCFileInputFormat;
+import org.apache.hadoop.hive.ql.io.RCFileOutputFormat;
+import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hcatalog.cli.SemanticAnalysis.HCatSemanticAnalyzer;
+import org.apache.hcatalog.common.HCatConstants;
+import org.apache.hcatalog.pig.HCatLoader;
+import org.apache.pig.ExecType;
+import org.apache.pig.PigServer;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.impl.util.UDFContext;
+import org.apache.thrift.TException;
+
+public class TestPigStorageDriver extends TestCase {
+
+  private HiveConf hcatConf;
+  private Driver hcatDriver;
+  private HiveMetaStoreClient msc;
+  private static String tblLocation = "/tmp/test_pig/data";
+  private static String anyExistingFileInCurDir = "ivy.xml";
+  private static String warehouseDir = "/tmp/hcat_junit_warehouse";
+
+  @Override
+  protected void setUp() throws Exception {
+
+    hcatConf = new HiveConf(this.getClass());
+    hcatConf.set(ConfVars.PREEXECHOOKS.varname, "");
+    hcatConf.set(ConfVars.POSTEXECHOOKS.varname, "");
+    hcatConf.set(ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
+    hcatConf.set(ConfVars.SEMANTIC_ANALYZER_HOOK.varname, HCatSemanticAnalyzer.class.getName());
+    hcatDriver = new Driver(hcatConf);
+    msc = new HiveMetaStoreClient(hcatConf);
+    SessionState.start(new CliSessionState(hcatConf));
+    super.setUp();
+  }
+
+  @Override
+  protected void tearDown() throws Exception {
+    super.tearDown();
+  }
+
+  public void testPigStorageDriver() throws IOException, CommandNeedRetryException{
+
+    String fsLoc = hcatConf.get("fs.default.name");
+    Path tblPath = new Path(fsLoc, tblLocation);
+    String tblName = "junit_pigstorage";
+    tblPath.getFileSystem(hcatConf).copyFromLocalFile(new Path(anyExistingFileInCurDir),tblPath);
+
+    hcatDriver.run("drop table " + tblName);
+    CommandProcessorResponse resp;
+    String createTable = "create table " + tblName + " (a string) partitioned by (b string)
stored as TEXTFILE";
+
+    resp = hcatDriver.run(createTable);
+    assertEquals(0, resp.getResponseCode());
+    assertNull(resp.getErrorMessage());
+
+    resp = hcatDriver.run("alter table " + tblName + " add partition (b='2010-10-10') location
'"+new Path(fsLoc, "/tmp/test_pig")+"'");
+    assertEquals(0, resp.getResponseCode());
+    assertNull(resp.getErrorMessage());
+
+    resp = hcatDriver.run("alter table " + tblName + " partition (b='2010-10-10') set fileformat
TEXTFILE");
+    assertEquals(0, resp.getResponseCode());
+    assertNull(resp.getErrorMessage());
+
+    resp =  hcatDriver.run("desc extended " + tblName + " partition (b='2010-10-10')");
+    assertEquals(0, resp.getResponseCode());
+    assertNull(resp.getErrorMessage());
+
+    PigServer server = new PigServer(ExecType.LOCAL, hcatConf.getAllProperties());
+    UDFContext.getUDFContext().setClientSystemProps();
+    server.registerQuery(" a = load '" + tblName + "' using "+HCatLoader.class.getName()+";");
+    Iterator<Tuple> itr = server.openIterator("a");
+    boolean result = compareWithFile(itr, anyExistingFileInCurDir, 2, "2010-10-10", null);
+    assertTrue(result);
+    
+    server.registerQuery("a = load '"+tblPath.toString()+"' using PigStorage() as (a:chararray);");
+    server.store("a", tblName, HCatStorer.class.getName() + "('b=2010-10-11')");
+    
+    server.registerQuery("a = load '" + warehouseDir + "/" + tblName + "/b=2010-10-11' using
PigStorage() as (a:chararray);");
+    itr = server.openIterator("a");
+    result = compareWithFile(itr, anyExistingFileInCurDir, 1, "2010-10-11", null);
+    assertTrue(result);
+    
+    // Test multi-store
+    server.registerQuery("a = load '"+tblPath.toString()+"' using PigStorage() as (a:chararray);");
+    server.registerQuery("store a into '" + tblName + "' using " +  HCatStorer.class.getName()
+ "('b=2010-11-01');");
+    server.registerQuery("store a into '" + tblName + "' using " +  HCatStorer.class.getName()
+ "('b=2010-11-02');");
+    
+    server.registerQuery("a = load '" + warehouseDir + "/" + tblName + "/b=2010-11-01' using
PigStorage() as (a:chararray);");
+    itr = server.openIterator("a");
+    result = compareWithFile(itr, anyExistingFileInCurDir, 1, "2010-11-01", null);
+    assertTrue(result);
+    
+    server.registerQuery("a = load '" + warehouseDir + "/" + tblName + "/b=2010-11-02' using
PigStorage() as (a:chararray);");
+    itr = server.openIterator("a");
+    result = compareWithFile(itr, anyExistingFileInCurDir, 1, "2010-11-02", null);
+    assertTrue(result);
+    
+    hcatDriver.run("drop table " + tblName);
+  }
+  
+  private boolean compareWithFile(Iterator<Tuple> itr, String factFile, int numColumn,
String key, String valueSuffix) throws IOException {
+      DataInputStream stream = new DataInputStream(new BufferedInputStream(new FileInputStream(new
File(factFile))));
+      while(itr.hasNext()){
+        Tuple t = itr.next();
+        assertEquals(numColumn, t.size());
+        if(t.get(0) != null) {
+          // If underlying data-field is empty. PigStorage inserts null instead
+          // of empty String objects.
+          assertTrue(t.get(0) instanceof String);
+          String expected = stream.readLine();
+          if (valueSuffix!=null)
+              expected += valueSuffix;
+          assertEquals(expected, t.get(0));
+        }
+        else{
+          assertTrue(stream.readLine().isEmpty());
+        }
+        
+        if (numColumn>1) {
+            // The second column must be key
+            assertTrue(t.get(1) instanceof String);
+            assertEquals(key, t.get(1));
+        }
+      }
+      assertEquals(0,stream.available());
+      stream.close();
+      return true;
+  }
+
+  public void testDelim() throws MetaException, TException, UnknownTableException, NoSuchObjectException,
InvalidOperationException, IOException, CommandNeedRetryException{
+
+    hcatDriver.run("drop table junit_pigstorage_delim");
+
+    CommandProcessorResponse resp;
+    String createTable = "create table junit_pigstorage_delim (a0 string, a1 string) partitioned
by (b string) stored as RCFILE";
+
+    resp = hcatDriver.run(createTable);
+
+    assertEquals(0, resp.getResponseCode());
+    assertNull(resp.getErrorMessage());
+
+    resp = hcatDriver.run("alter table junit_pigstorage_delim add partition (b='2010-10-10')");
+    assertEquals(0, resp.getResponseCode());
+    assertNull(resp.getErrorMessage());
+
+    resp = hcatDriver.run("alter table junit_pigstorage_delim partition (b='2010-10-10')
set fileformat TEXTFILE");
+
+    Partition part = msc.getPartition(MetaStoreUtils.DEFAULT_DATABASE_NAME, "junit_pigstorage_delim",
"b=2010-10-10");
+    Map<String,String> partParms = part.getParameters();
+    partParms.put(HCatConstants.HCAT_PIG_LOADER_ARGS, "control-A");
+    partParms.put(HCatConstants.HCAT_PIG_STORER_ARGS, "control-A");
+
+    msc.alter_partition(MetaStoreUtils.DEFAULT_DATABASE_NAME, "junit_pigstorage_delim", part);
+
+    PigServer server = new PigServer(ExecType.LOCAL, hcatConf.getAllProperties());
+    UDFContext.getUDFContext().setClientSystemProps();
+    server.registerQuery(" a = load 'junit_pigstorage_delim' using "+HCatLoader.class.getName()+";");
+    try{
+      server.openIterator("a");
+    }catch(FrontendException fe){}
+    
+    resp = hcatDriver.run("alter table junit_pigstorage_delim set fileformat TEXTFILE");
+    assertEquals(0, resp.getResponseCode());
+    assertNull(resp.getErrorMessage());
+    resp = hcatDriver.run("alter table junit_pigstorage_delim set TBLPROPERTIES ('hcat.pig.loader.args'=':',
'hcat.pig.storer.args'=':')");
+    assertEquals(0, resp.getResponseCode());
+    assertNull(resp.getErrorMessage());
+    
+    File inputFile = File.createTempFile("hcat_test", "");
+    PrintWriter p = new PrintWriter(new FileWriter(inputFile));
+    p.println("1\t2");
+    p.println("3\t4");
+    p.close();
+    server.registerQuery("a = load '"+inputFile.toString()+"' using PigStorage() as (a0:chararray,
a1:chararray);");
+    server.store("a", "junit_pigstorage_delim", HCatStorer.class.getName() + "('b=2010-10-11')");
+    
+    server.registerQuery("a = load '/tmp/hcat_junit_warehouse/junit_pigstorage_delim/b=2010-10-11'
using PigStorage() as (a:chararray);");
+    Iterator<Tuple> itr = server.openIterator("a");
+    
+    assertTrue(itr.hasNext());
+    Tuple t = itr.next();
+    assertTrue(t.get(0).equals("1:2"));
+    
+    assertTrue(itr.hasNext());
+    t = itr.next();
+    assertTrue(t.get(0).equals("3:4"));
+    
+    assertFalse(itr.hasNext());
+    inputFile.delete();
+  }
+  
+  public void testMultiConstructArgs() throws MetaException, TException, UnknownTableException,
NoSuchObjectException, InvalidOperationException, IOException, CommandNeedRetryException{
+
+      String fsLoc = hcatConf.get("fs.default.name");
+      Path tblPath = new Path(fsLoc, tblLocation);
+      String tblName = "junit_pigstorage_constructs";
+      tblPath.getFileSystem(hcatConf).copyFromLocalFile(new Path(anyExistingFileInCurDir),tblPath);
+
+      hcatDriver.run("drop table junit_pigstorage_constructs");
+
+      CommandProcessorResponse resp;
+      String createTable = "create table " + tblName + " (a string) partitioned by (b string)
stored as TEXTFILE";
+
+      resp = hcatDriver.run(createTable);
+
+      assertEquals(0, resp.getResponseCode());
+      assertNull(resp.getErrorMessage());
+
+      resp = hcatDriver.run("alter table " + tblName + " set TBLPROPERTIES ('hcat.pig.storer'='org.apache.hcatalog.pig.MyPigStorage',
'hcat.pig.storer.args'=':#hello', 'hcat.pig.args.delimiter'='#')");
+      assertEquals(0, resp.getResponseCode());
+      assertNull(resp.getErrorMessage());
+      
+      PigServer server = new PigServer(ExecType.LOCAL, hcatConf.getAllProperties());
+      UDFContext.getUDFContext().setClientSystemProps();
+      
+      server.registerQuery("a = load '"+tblPath.toString()+"' using PigStorage() as (a:chararray);");
+      server.store("a", tblName, HCatStorer.class.getName() + "('b=2010-10-11')");
+      
+      server.registerQuery("a = load '" + warehouseDir + "/" + tblName + "/b=2010-10-11'
using PigStorage() as (a:chararray);");
+      Iterator<Tuple> itr = server.openIterator("a");
+      boolean result = compareWithFile(itr, anyExistingFileInCurDir, 1, "2010-10-11", ":hello");
+      assertTrue(result);
+    }
+}

Modified: incubator/hcatalog/trunk/src/test/org/apache/hcatalog/rcfile/TestRCFileOutputStorageDriver.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/org/apache/hcatalog/rcfile/TestRCFileOutputStorageDriver.java?rev=1244334&r1=1244333&r2=1244334&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/org/apache/hcatalog/rcfile/TestRCFileOutputStorageDriver.java
(original)
+++ incubator/hcatalog/trunk/src/test/org/apache/hcatalog/rcfile/TestRCFileOutputStorageDriver.java
Wed Feb 15 03:53:50 2012
@@ -45,7 +45,7 @@ public class TestRCFileOutputStorageDriv
   public void testConversion() throws IOException {
     Configuration conf = new Configuration();
     JobContext jc = HCatHadoopShims.Instance.get().createJobContext(conf, new JobID());
-    String jobString = HCatUtil.serialize(OutputJobInfo.create(null,null,null,null,null));
+    String jobString = HCatUtil.serialize(OutputJobInfo.create(null,null,null));
     jc.getConfiguration().set(HCatConstants.HCAT_KEY_OUTPUT_INFO,jobString);
 
     HCatSchema schema = buildHiveSchema();

Modified: incubator/hcatalog/trunk/storage-drivers/build.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-drivers/build.xml?rev=1244334&r1=1244333&r2=1244334&view=diff
==============================================================================
--- incubator/hcatalog/trunk/storage-drivers/build.xml (original)
+++ incubator/hcatalog/trunk/storage-drivers/build.xml Wed Feb 15 03:53:50 2012
@@ -24,8 +24,10 @@
             <isset property="dist.drivers.dir"/>
         </condition>
         <!-- add storage drivers here -->
+<!--  TODO migrate hbase storageHandler
         <echo>Executing storage-driver &quot;${target}&quot; for hbase</echo>
         <ant target="${target}" dir="hbase" inheritAll="false" useNativeBasedir="true"/>
+-->
     </target>
 
 



Mime
View raw message