From hcatalog-commits-return-949-apmail-incubator-hcatalog-commits-archive=incubator.apache.org@incubator.apache.org Sun Jun 10 07:09:54 2012 Return-Path: X-Original-To: apmail-incubator-hcatalog-commits-archive@minotaur.apache.org Delivered-To: apmail-incubator-hcatalog-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 37AB3979E for ; Sun, 10 Jun 2012 07:09:54 +0000 (UTC) Received: (qmail 88858 invoked by uid 500); 10 Jun 2012 07:09:54 -0000 Delivered-To: apmail-incubator-hcatalog-commits-archive@incubator.apache.org Received: (qmail 88742 invoked by uid 500); 10 Jun 2012 07:09:51 -0000 Mailing-List: contact hcatalog-commits-help@incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hcatalog-dev@incubator.apache.org Delivered-To: mailing list hcatalog-commits@incubator.apache.org Received: (qmail 88709 invoked by uid 99); 10 Jun 2012 07:09:49 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Sun, 10 Jun 2012 07:09:49 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Sun, 10 Jun 2012 07:09:46 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id EAFBE238899C; Sun, 10 Jun 2012 07:09:24 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1348538 - in /incubator/hcatalog/trunk: CHANGES.txt src/java/org/apache/hcatalog/common/HCatUtil.java src/java/org/apache/hcatalog/data/schema/HCatSchema.java src/test/org/apache/hcatalog/common/TestHCatUtil.java Date: Sun, 10 Jun 2012 07:09:24 -0000 To: hcatalog-commits@incubator.apache.org From: khorgath@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20120610070924.EAFBE238899C@eris.apache.org> Author: khorgath Date: Sun Jun 10 07:09:24 2012 New Revision: 1348538 URL: http://svn.apache.org/viewvc?rev=1348538&view=rev Log: HCATALOG-423 HCatalog should include SerDe-reported fields in the table schema (traviscrawford via khorgath) Modified: incubator/hcatalog/trunk/CHANGES.txt incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatUtil.java incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatSchema.java incubator/hcatalog/trunk/src/test/org/apache/hcatalog/common/TestHCatUtil.java Modified: incubator/hcatalog/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/CHANGES.txt?rev=1348538&r1=1348537&r2=1348538&view=diff ============================================================================== --- incubator/hcatalog/trunk/CHANGES.txt (original) +++ incubator/hcatalog/trunk/CHANGES.txt Sun Jun 10 07:09:24 2012 @@ -45,6 +45,8 @@ Trunk (unreleased changes) OPTIMIZATIONS BUG FIXES + HCAT-423 HCatalog should include SerDe-reported fields in the table schema (traviscrawford via khorgath) + HCAT-362 add --local-infile to the mysql command (arpitgupta via daijy) HCAT-417 List Ant 1.8 in install pre-reqs (jghoman via daijy) Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatUtil.java URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatUtil.java?rev=1348538&r1=1348537&r2=1348538&view=diff ============================================================================== --- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatUtil.java (original) +++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatUtil.java Sun Jun 10 07:09:24 2012 @@ -184,10 +184,9 @@ public class HCatUtil { return client.getTable(dbName, tableName); } - public static HCatSchema getTableSchemaWithPtnCols(Table table) - throws IOException { - HCatSchema tableSchema = extractSchemaFromStorageDescriptor(table - .getSd()); + public static HCatSchema getTableSchemaWithPtnCols(Table table) throws IOException { + HCatSchema tableSchema = new HCatSchema(HCatUtil.getHCatFieldSchemaList( + new org.apache.hadoop.hive.ql.metadata.Table(table).getCols())); if (table.getPartitionKeys().size() != 0) { Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatSchema.java URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatSchema.java?rev=1348538&r1=1348537&r2=1348538&view=diff ============================================================================== --- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatSchema.java (original) +++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatSchema.java Sun Jun 10 07:09:24 2012 @@ -157,4 +157,15 @@ public class HCatSchema implements Seria } return sb.toString(); } + + @Override + public boolean equals(Object other) { + return !((other == null) || !(other instanceof HCatSchema)) + && this.hashCode() == other.hashCode(); + } + + @Override + public int hashCode() { + return toString().hashCode(); + } } Modified: incubator/hcatalog/trunk/src/test/org/apache/hcatalog/common/TestHCatUtil.java URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/org/apache/hcatalog/common/TestHCatUtil.java?rev=1348538&r1=1348537&r2=1348538&view=diff ============================================================================== --- incubator/hcatalog/trunk/src/test/org/apache/hcatalog/common/TestHCatUtil.java (original) +++ incubator/hcatalog/trunk/src/test/org/apache/hcatalog/common/TestHCatUtil.java Sun Jun 10 07:09:24 2012 @@ -17,17 +17,31 @@ */ package org.apache.hcatalog.common; +import java.io.IOException; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; +import java.util.Map; -import junit.framework.TestCase; - +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hcatalog.common.HCatUtil; - -public class TestHCatUtil extends TestCase{ +import org.apache.hadoop.hive.metastore.TableType; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.Order; +import org.apache.hadoop.hive.metastore.api.SerDeInfo; +import org.apache.hadoop.hive.metastore.api.StorageDescriptor; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.serde.Constants; +import org.apache.hcatalog.data.schema.HCatFieldSchema; +import org.apache.hcatalog.data.schema.HCatSchema; +import org.junit.Assert; +import org.junit.Test; +public class TestHCatUtil { + @Test public void testFsPermissionOperation(){ HashMap permsCode = new HashMap(); @@ -43,7 +57,7 @@ public class TestHCatUtil extends TestCa Integer code = (((i*8)+j)*8)+k; String perms = (new FsPermission(Short.decode(sb.toString()))).toString(); if (permsCode.containsKey(perms)){ - assertEquals("permissions(" + perms + ") mapped to multiple codes",code,permsCode.get(perms)); + Assert.assertEquals("permissions(" + perms + ") mapped to multiple codes", code, permsCode.get(perms)); } permsCode.put(perms, code); assertFsPermissionTransformationIsGood(perms); @@ -53,9 +67,10 @@ public class TestHCatUtil extends TestCa } private void assertFsPermissionTransformationIsGood(String perms) { - assertEquals(perms,FsPermission.valueOf("-"+perms).toString()); + Assert.assertEquals(perms, FsPermission.valueOf("-" + perms).toString()); } + @Test public void testValidateMorePermissive(){ assertConsistentFsPermissionBehaviour(FsAction.ALL,true,true,true,true,true,true,true,true); assertConsistentFsPermissionBehaviour(FsAction.READ,false,true,false,true,false,false,false,false); @@ -73,27 +88,91 @@ public class TestHCatUtil extends TestCa boolean versusX, boolean versusR, boolean versusW, boolean versusRX, boolean versusRW, boolean versusWX){ - assertTrue(versusAll == HCatUtil.validateMorePermissive(base, FsAction.ALL)); - assertTrue(versusX == HCatUtil.validateMorePermissive(base, FsAction.EXECUTE)); - assertTrue(versusNone == HCatUtil.validateMorePermissive(base, FsAction.NONE)); - assertTrue(versusR == HCatUtil.validateMorePermissive(base, FsAction.READ)); - assertTrue(versusRX == HCatUtil.validateMorePermissive(base, FsAction.READ_EXECUTE)); - assertTrue(versusRW == HCatUtil.validateMorePermissive(base, FsAction.READ_WRITE)); - assertTrue(versusW == HCatUtil.validateMorePermissive(base, FsAction.WRITE)); - assertTrue(versusWX == HCatUtil.validateMorePermissive(base, FsAction.WRITE_EXECUTE)); + Assert.assertTrue(versusAll == HCatUtil.validateMorePermissive(base, FsAction.ALL)); + Assert.assertTrue(versusX == HCatUtil.validateMorePermissive(base, FsAction.EXECUTE)); + Assert.assertTrue(versusNone == HCatUtil.validateMorePermissive(base, FsAction.NONE)); + Assert.assertTrue(versusR == HCatUtil.validateMorePermissive(base, FsAction.READ)); + Assert.assertTrue(versusRX == HCatUtil.validateMorePermissive(base, FsAction.READ_EXECUTE)); + Assert.assertTrue(versusRW == HCatUtil.validateMorePermissive(base, FsAction.READ_WRITE)); + Assert.assertTrue(versusW == HCatUtil.validateMorePermissive(base, FsAction.WRITE)); + Assert.assertTrue(versusWX == HCatUtil.validateMorePermissive(base, FsAction.WRITE_EXECUTE)); } + @Test public void testExecutePermissionsCheck(){ - assertTrue(HCatUtil.validateExecuteBitPresentIfReadOrWrite(FsAction.ALL)); - assertTrue(HCatUtil.validateExecuteBitPresentIfReadOrWrite(FsAction.NONE)); - assertTrue(HCatUtil.validateExecuteBitPresentIfReadOrWrite(FsAction.EXECUTE)); - assertTrue(HCatUtil.validateExecuteBitPresentIfReadOrWrite(FsAction.READ_EXECUTE)); - assertTrue(HCatUtil.validateExecuteBitPresentIfReadOrWrite(FsAction.WRITE_EXECUTE)); - - assertFalse(HCatUtil.validateExecuteBitPresentIfReadOrWrite(FsAction.READ)); - assertFalse(HCatUtil.validateExecuteBitPresentIfReadOrWrite(FsAction.WRITE)); - assertFalse(HCatUtil.validateExecuteBitPresentIfReadOrWrite(FsAction.READ_WRITE)); + Assert.assertTrue(HCatUtil.validateExecuteBitPresentIfReadOrWrite(FsAction.ALL)); + Assert.assertTrue(HCatUtil.validateExecuteBitPresentIfReadOrWrite(FsAction.NONE)); + Assert.assertTrue(HCatUtil.validateExecuteBitPresentIfReadOrWrite(FsAction.EXECUTE)); + Assert.assertTrue(HCatUtil.validateExecuteBitPresentIfReadOrWrite(FsAction.READ_EXECUTE)); + Assert.assertTrue(HCatUtil.validateExecuteBitPresentIfReadOrWrite(FsAction.WRITE_EXECUTE)); + + Assert.assertFalse(HCatUtil.validateExecuteBitPresentIfReadOrWrite(FsAction.READ)); + Assert.assertFalse(HCatUtil.validateExecuteBitPresentIfReadOrWrite(FsAction.WRITE)); + Assert.assertFalse(HCatUtil.validateExecuteBitPresentIfReadOrWrite(FsAction.READ_WRITE)); + + } + @Test + public void testGetTableSchemaWithPtnColsApi() throws IOException { + // Check the schema of a table with one field & no partition keys. + StorageDescriptor sd = new StorageDescriptor( + Lists.newArrayList(new FieldSchema("username", Constants.STRING_TYPE_NAME, null)), + "location", "org.apache.hadoop.mapred.TextInputFormat", + "org.apache.hadoop.mapred.TextOutputFormat", false, -1, new SerDeInfo(), + new ArrayList(), new ArrayList(), new HashMap()); + Table table = new Table("test_tblname", "test_dbname", "test_owner", 0, 0, 0, + sd, new ArrayList(), new HashMap(), + "viewOriginalText", "viewExpandedText", TableType.EXTERNAL_TABLE.name()); + + List expectedHCatSchema = + Lists.newArrayList(new HCatFieldSchema("username", HCatFieldSchema.Type.STRING, null)); + + Assert.assertEquals(new HCatSchema(expectedHCatSchema), + HCatUtil.getTableSchemaWithPtnCols(table)); + + // Add a partition key & ensure its reflected in the schema. + List partitionKeys = + Lists.newArrayList(new FieldSchema("dt", Constants.STRING_TYPE_NAME, null)); + table.setPartitionKeys(partitionKeys); + expectedHCatSchema.add(new HCatFieldSchema("dt", HCatFieldSchema.Type.STRING, null)); + Assert.assertEquals(new HCatSchema(expectedHCatSchema), + HCatUtil.getTableSchemaWithPtnCols(table)); } + /** + * Hive represents tables in two ways: + *
    + *
  • org.apache.hadoop.hive.metastore.api.Table - exactly whats stored in the metastore
    • + *
  • org.apache.hadoop.hive.ql.metadata.Table - adds business logic over api.Table
    • + *
+ * Here we check SerDe-reported fields are included in the table schema. + */ + @Test + public void testGetTableSchemaWithPtnColsSerDeReportedFields() throws IOException { + Map parameters = Maps.newHashMap(); + parameters.put(Constants.SERIALIZATION_CLASS, + "org.apache.hadoop.hive.serde2.thrift.test.IntString"); + parameters.put(Constants.SERIALIZATION_FORMAT, "org.apache.thrift.protocol.TBinaryProtocol"); + + SerDeInfo serDeInfo = new SerDeInfo(null, + "org.apache.hadoop.hive.serde2.thrift.ThriftDeserializer", parameters); + + // StorageDescriptor has an empty list of fields - SerDe will report them. + StorageDescriptor sd = new StorageDescriptor(new ArrayList(), "location", + "org.apache.hadoop.mapred.TextInputFormat", "org.apache.hadoop.mapred.TextOutputFormat", + false, -1, serDeInfo, new ArrayList(), new ArrayList(), + new HashMap()); + + Table table = new Table("test_tblname", "test_dbname", "test_owner", 0, 0, 0, + sd, new ArrayList(), new HashMap(), + "viewOriginalText", "viewExpandedText", TableType.EXTERNAL_TABLE.name()); + + List expectedHCatSchema = Lists.newArrayList( + new HCatFieldSchema("myint", HCatFieldSchema.Type.INT, null), + new HCatFieldSchema("mystring", HCatFieldSchema.Type.STRING, null), + new HCatFieldSchema("underscore_int", HCatFieldSchema.Type.INT, null)); + + Assert.assertEquals(new HCatSchema(expectedHCatSchema), + HCatUtil.getTableSchemaWithPtnCols(table)); + } }