incubator-hcatalog-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From khorg...@apache.org
Subject svn commit: r1348538 - in /incubator/hcatalog/trunk: CHANGES.txt src/java/org/apache/hcatalog/common/HCatUtil.java src/java/org/apache/hcatalog/data/schema/HCatSchema.java src/test/org/apache/hcatalog/common/TestHCatUtil.java
Date Sun, 10 Jun 2012 07:09:24 GMT
Author: khorgath
Date: Sun Jun 10 07:09:24 2012
New Revision: 1348538

URL: http://svn.apache.org/viewvc?rev=1348538&view=rev
Log:
HCATALOG-423 HCatalog should include SerDe-reported fields in the table schema (traviscrawford
via khorgath)

Modified:
    incubator/hcatalog/trunk/CHANGES.txt
    incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatUtil.java
    incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatSchema.java
    incubator/hcatalog/trunk/src/test/org/apache/hcatalog/common/TestHCatUtil.java

Modified: incubator/hcatalog/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/CHANGES.txt?rev=1348538&r1=1348537&r2=1348538&view=diff
==============================================================================
--- incubator/hcatalog/trunk/CHANGES.txt (original)
+++ incubator/hcatalog/trunk/CHANGES.txt Sun Jun 10 07:09:24 2012
@@ -45,6 +45,8 @@ Trunk (unreleased changes)
   OPTIMIZATIONS
 
   BUG FIXES
+  HCAT-423 HCatalog should include SerDe-reported fields in the table schema (traviscrawford
via khorgath)
+
   HCAT-362 add --local-infile to the mysql command (arpitgupta via daijy)
 
   HCAT-417 List Ant 1.8 in install pre-reqs (jghoman via daijy)

Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatUtil.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatUtil.java?rev=1348538&r1=1348537&r2=1348538&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatUtil.java (original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatUtil.java Sun Jun 10
07:09:24 2012
@@ -184,10 +184,9 @@ public class HCatUtil {
         return client.getTable(dbName, tableName);
     }
 
-    public static HCatSchema getTableSchemaWithPtnCols(Table table)
-            throws IOException {
-        HCatSchema tableSchema = extractSchemaFromStorageDescriptor(table
-                .getSd());
+    public static HCatSchema getTableSchemaWithPtnCols(Table table) throws IOException {
+        HCatSchema tableSchema = new HCatSchema(HCatUtil.getHCatFieldSchemaList(
+                new org.apache.hadoop.hive.ql.metadata.Table(table).getCols()));
 
         if (table.getPartitionKeys().size() != 0) {
 

Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatSchema.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatSchema.java?rev=1348538&r1=1348537&r2=1348538&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatSchema.java (original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatSchema.java Sun
Jun 10 07:09:24 2012
@@ -157,4 +157,15 @@ public class HCatSchema implements Seria
         }
         return sb.toString();
     }
+
+    @Override
+    public boolean equals(Object other) {
+        return !((other == null) || !(other instanceof HCatSchema))
+            && this.hashCode() == other.hashCode();
+    }
+
+    @Override
+    public int hashCode() {
+        return toString().hashCode();
+    }
 }

Modified: incubator/hcatalog/trunk/src/test/org/apache/hcatalog/common/TestHCatUtil.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/org/apache/hcatalog/common/TestHCatUtil.java?rev=1348538&r1=1348537&r2=1348538&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/org/apache/hcatalog/common/TestHCatUtil.java (original)
+++ incubator/hcatalog/trunk/src/test/org/apache/hcatalog/common/TestHCatUtil.java Sun Jun
10 07:09:24 2012
@@ -17,17 +17,31 @@
  */
 package org.apache.hcatalog.common;
 
+import java.io.IOException;
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 
-import junit.framework.TestCase;
-
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
 import org.apache.hadoop.fs.permission.FsAction;
 import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hcatalog.common.HCatUtil;
-
-public class TestHCatUtil extends TestCase{
+import org.apache.hadoop.hive.metastore.TableType;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.Order;
+import org.apache.hadoop.hive.metastore.api.SerDeInfo;
+import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.serde.Constants;
+import org.apache.hcatalog.data.schema.HCatFieldSchema;
+import org.apache.hcatalog.data.schema.HCatSchema;
+import org.junit.Assert;
+import org.junit.Test;
 
+public class TestHCatUtil {
 
+  @Test
   public void testFsPermissionOperation(){
 
     HashMap<String,Integer> permsCode = new HashMap<String,Integer>();
@@ -43,7 +57,7 @@ public class TestHCatUtil extends TestCa
           Integer code = (((i*8)+j)*8)+k;
           String perms = (new FsPermission(Short.decode(sb.toString()))).toString();
           if (permsCode.containsKey(perms)){
-            assertEquals("permissions(" + perms + ") mapped to multiple codes",code,permsCode.get(perms));
+            Assert.assertEquals("permissions(" + perms + ") mapped to multiple codes", code,
permsCode.get(perms));
           }
           permsCode.put(perms, code);
           assertFsPermissionTransformationIsGood(perms);
@@ -53,9 +67,10 @@ public class TestHCatUtil extends TestCa
   }
 
   private void assertFsPermissionTransformationIsGood(String perms) {
-    assertEquals(perms,FsPermission.valueOf("-"+perms).toString());
+    Assert.assertEquals(perms, FsPermission.valueOf("-" + perms).toString());
   }
 
+  @Test
   public void testValidateMorePermissive(){
     assertConsistentFsPermissionBehaviour(FsAction.ALL,true,true,true,true,true,true,true,true);
     assertConsistentFsPermissionBehaviour(FsAction.READ,false,true,false,true,false,false,false,false);
@@ -73,27 +88,91 @@ public class TestHCatUtil extends TestCa
       boolean versusX, boolean versusR, boolean versusW,
       boolean versusRX, boolean versusRW,  boolean versusWX){
 
-    assertTrue(versusAll == HCatUtil.validateMorePermissive(base, FsAction.ALL));
-    assertTrue(versusX == HCatUtil.validateMorePermissive(base, FsAction.EXECUTE));
-    assertTrue(versusNone == HCatUtil.validateMorePermissive(base, FsAction.NONE));
-    assertTrue(versusR == HCatUtil.validateMorePermissive(base, FsAction.READ));
-    assertTrue(versusRX == HCatUtil.validateMorePermissive(base, FsAction.READ_EXECUTE));
-    assertTrue(versusRW == HCatUtil.validateMorePermissive(base, FsAction.READ_WRITE));
-    assertTrue(versusW == HCatUtil.validateMorePermissive(base, FsAction.WRITE));
-    assertTrue(versusWX == HCatUtil.validateMorePermissive(base, FsAction.WRITE_EXECUTE));
+    Assert.assertTrue(versusAll == HCatUtil.validateMorePermissive(base, FsAction.ALL));
+    Assert.assertTrue(versusX == HCatUtil.validateMorePermissive(base, FsAction.EXECUTE));
+    Assert.assertTrue(versusNone == HCatUtil.validateMorePermissive(base, FsAction.NONE));
+    Assert.assertTrue(versusR == HCatUtil.validateMorePermissive(base, FsAction.READ));
+    Assert.assertTrue(versusRX == HCatUtil.validateMorePermissive(base, FsAction.READ_EXECUTE));
+    Assert.assertTrue(versusRW == HCatUtil.validateMorePermissive(base, FsAction.READ_WRITE));
+    Assert.assertTrue(versusW == HCatUtil.validateMorePermissive(base, FsAction.WRITE));
+    Assert.assertTrue(versusWX == HCatUtil.validateMorePermissive(base, FsAction.WRITE_EXECUTE));
   }
 
+  @Test
   public void testExecutePermissionsCheck(){
-    assertTrue(HCatUtil.validateExecuteBitPresentIfReadOrWrite(FsAction.ALL));
-    assertTrue(HCatUtil.validateExecuteBitPresentIfReadOrWrite(FsAction.NONE));
-    assertTrue(HCatUtil.validateExecuteBitPresentIfReadOrWrite(FsAction.EXECUTE));
-    assertTrue(HCatUtil.validateExecuteBitPresentIfReadOrWrite(FsAction.READ_EXECUTE));
-    assertTrue(HCatUtil.validateExecuteBitPresentIfReadOrWrite(FsAction.WRITE_EXECUTE));
-
-    assertFalse(HCatUtil.validateExecuteBitPresentIfReadOrWrite(FsAction.READ));
-    assertFalse(HCatUtil.validateExecuteBitPresentIfReadOrWrite(FsAction.WRITE));
-    assertFalse(HCatUtil.validateExecuteBitPresentIfReadOrWrite(FsAction.READ_WRITE));
+    Assert.assertTrue(HCatUtil.validateExecuteBitPresentIfReadOrWrite(FsAction.ALL));
+    Assert.assertTrue(HCatUtil.validateExecuteBitPresentIfReadOrWrite(FsAction.NONE));
+    Assert.assertTrue(HCatUtil.validateExecuteBitPresentIfReadOrWrite(FsAction.EXECUTE));
+    Assert.assertTrue(HCatUtil.validateExecuteBitPresentIfReadOrWrite(FsAction.READ_EXECUTE));
+    Assert.assertTrue(HCatUtil.validateExecuteBitPresentIfReadOrWrite(FsAction.WRITE_EXECUTE));
+
+    Assert.assertFalse(HCatUtil.validateExecuteBitPresentIfReadOrWrite(FsAction.READ));
+    Assert.assertFalse(HCatUtil.validateExecuteBitPresentIfReadOrWrite(FsAction.WRITE));
+    Assert.assertFalse(HCatUtil.validateExecuteBitPresentIfReadOrWrite(FsAction.READ_WRITE));
+
+  }
 
+  @Test
+  public void testGetTableSchemaWithPtnColsApi() throws IOException {
+    // Check the schema of a table with one field & no partition keys.
+    StorageDescriptor sd = new StorageDescriptor(
+        Lists.newArrayList(new FieldSchema("username", Constants.STRING_TYPE_NAME, null)),
+        "location", "org.apache.hadoop.mapred.TextInputFormat",
+        "org.apache.hadoop.mapred.TextOutputFormat", false, -1, new SerDeInfo(),
+        new ArrayList<String>(), new ArrayList<Order>(), new HashMap<String,
String>());
+    Table table = new Table("test_tblname", "test_dbname", "test_owner", 0, 0, 0,
+        sd, new ArrayList<FieldSchema>(), new HashMap<String, String>(),
+        "viewOriginalText", "viewExpandedText", TableType.EXTERNAL_TABLE.name());
+
+    List<HCatFieldSchema> expectedHCatSchema =
+        Lists.newArrayList(new HCatFieldSchema("username", HCatFieldSchema.Type.STRING, null));
+
+    Assert.assertEquals(new HCatSchema(expectedHCatSchema),
+        HCatUtil.getTableSchemaWithPtnCols(table));
+
+    // Add a partition key & ensure its reflected in the schema.
+    List<FieldSchema> partitionKeys =
+        Lists.newArrayList(new FieldSchema("dt", Constants.STRING_TYPE_NAME, null));
+    table.setPartitionKeys(partitionKeys);
+    expectedHCatSchema.add(new HCatFieldSchema("dt", HCatFieldSchema.Type.STRING, null));
+    Assert.assertEquals(new HCatSchema(expectedHCatSchema),
+        HCatUtil.getTableSchemaWithPtnCols(table));
   }
 
+  /**
+   * Hive represents tables in two ways:
+   * <ul>
+   *   <li>org.apache.hadoop.hive.metastore.api.Table - exactly whats stored in the
metastore</li>
+   *   <li>org.apache.hadoop.hive.ql.metadata.Table - adds business logic over api.Table</li>
+   * </ul>
+   * Here we check SerDe-reported fields are included in the table schema.
+   */
+  @Test
+  public void testGetTableSchemaWithPtnColsSerDeReportedFields() throws IOException {
+    Map<String, String> parameters = Maps.newHashMap();
+    parameters.put(Constants.SERIALIZATION_CLASS,
+        "org.apache.hadoop.hive.serde2.thrift.test.IntString");
+    parameters.put(Constants.SERIALIZATION_FORMAT, "org.apache.thrift.protocol.TBinaryProtocol");
+
+    SerDeInfo serDeInfo = new SerDeInfo(null,
+        "org.apache.hadoop.hive.serde2.thrift.ThriftDeserializer", parameters);
+
+    // StorageDescriptor has an empty list of fields - SerDe will report them.
+    StorageDescriptor sd = new StorageDescriptor(new ArrayList<FieldSchema>(), "location",
+        "org.apache.hadoop.mapred.TextInputFormat", "org.apache.hadoop.mapred.TextOutputFormat",
+        false, -1, serDeInfo, new ArrayList<String>(), new ArrayList<Order>(),
+        new HashMap<String, String>());
+
+    Table table = new Table("test_tblname", "test_dbname", "test_owner", 0, 0, 0,
+        sd, new ArrayList<FieldSchema>(), new HashMap<String, String>(),
+        "viewOriginalText", "viewExpandedText", TableType.EXTERNAL_TABLE.name());
+
+    List<HCatFieldSchema> expectedHCatSchema = Lists.newArrayList(
+        new HCatFieldSchema("myint", HCatFieldSchema.Type.INT, null),
+        new HCatFieldSchema("mystring", HCatFieldSchema.Type.STRING, null),
+        new HCatFieldSchema("underscore_int", HCatFieldSchema.Type.INT, null));
+
+    Assert.assertEquals(new HCatSchema(expectedHCatSchema),
+        HCatUtil.getTableSchemaWithPtnCols(table));
+  }
 }



Mime
View raw message