incubator-hcatalog-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tra...@apache.org
Subject svn commit: r1379061 - in /incubator/hcatalog/trunk: ./ hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/ src/java/org/apache/hcatalog/common/ src/java/org/apache/hcatalog/mapreduce/ src/test/org/apache/hcatalog/common/
Date Thu, 30 Aug 2012 18:53:53 GMT
Author: travis
Date: Thu Aug 30 18:53:52 2012
New Revision: 1379061

URL: http://svn.apache.org/viewvc?rev=1379061&view=rev
Log:
HCAT-484 HCatalog should use ql.metadata Table and Partition classes (traviscrawford)

Modified:
    incubator/hcatalog/trunk/CHANGES.txt
    incubator/hcatalog/trunk/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/HCatLoader.java
    incubator/hcatalog/trunk/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/PigHCatUtil.java
    incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatUtil.java
    incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java
    incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileOutputFormatContainer.java
    incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/HCatBaseOutputFormat.java
    incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/HCatOutputFormat.java
    incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/HCatTableInfo.java
    incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/InitializeInput.java
    incubator/hcatalog/trunk/src/test/org/apache/hcatalog/common/TestHCatUtil.java

Modified: incubator/hcatalog/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/CHANGES.txt?rev=1379061&r1=1379060&r2=1379061&view=diff
==============================================================================
--- incubator/hcatalog/trunk/CHANGES.txt (original)
+++ incubator/hcatalog/trunk/CHANGES.txt Thu Aug 30 18:53:52 2012
@@ -38,6 +38,8 @@ Trunk (unreleased changes)
   HCAT-427 Document storage-based authorization (lefty via gates)
 
   IMPROVEMENTS
+  HCAT-484 HCatalog should use ql.metadata Table and Partition classes (traviscrawford)
+
   HCAT-485 Document that storage-based security ignores GRANT/REVOKE statements (lefty via
khorgath)
 
   HCAT-442 Documentation needs update for using HCatalog with pig (lefty via gates)

Modified: incubator/hcatalog/trunk/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/HCatLoader.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/HCatLoader.java?rev=1379061&r1=1379060&r2=1379061&view=diff
==============================================================================
--- incubator/hcatalog/trunk/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/HCatLoader.java
(original)
+++ incubator/hcatalog/trunk/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/HCatLoader.java
Thu Aug 30 18:53:52 2012
@@ -27,7 +27,7 @@ import java.util.Properties;
 
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.ql.metadata.Table;
 import org.apache.hadoop.mapreduce.InputFormat;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.security.Credentials;

Modified: incubator/hcatalog/trunk/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/PigHCatUtil.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/PigHCatUtil.java?rev=1379061&r1=1379060&r2=1379061&view=diff
==============================================================================
--- incubator/hcatalog/trunk/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/PigHCatUtil.java
(original)
+++ incubator/hcatalog/trunk/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/PigHCatUtil.java
Thu Aug 30 18:53:52 2012
@@ -32,7 +32,7 @@ import org.apache.hadoop.hive.conf.HiveC
 import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
 import org.apache.hadoop.hive.metastore.MetaStoreUtils;
 import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
-import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.ql.metadata.Table;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hcatalog.common.HCatConstants;
 import org.apache.hcatalog.common.HCatException;
@@ -141,7 +141,7 @@ public class PigHCatUtil {
     HiveMetaStoreClient client = null;
     try {
       client = getHiveMetaClient(hcatServerUri, hcatServerPrincipal, PigHCatUtil.class);
-      table = client.getTable(dbName, tableName);
+      table = HCatUtil.getTable(client, dbName, tableName);
     } catch (NoSuchObjectException nsoe){
       throw new PigException("Table not found : " + nsoe.getMessage(), PIG_EXCEPTION_CODE);
// prettier error messages to frontend
     } catch (Exception e) {

Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatUtil.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatUtil.java?rev=1379061&r1=1379060&r2=1379061&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatUtil.java (original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatUtil.java Thu Aug 30
18:53:52 2012
@@ -39,9 +39,11 @@ import org.apache.hadoop.hive.metastore.
 import org.apache.hadoop.hive.metastore.MetaStoreUtils;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
 import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
 import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
-import org.apache.hadoop.hive.metastore.api.Table;
 import org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat;
+import org.apache.hadoop.hive.ql.metadata.Partition;
+import org.apache.hadoop.hive.ql.metadata.Table;
 import org.apache.hadoop.hive.ql.plan.TableDesc;
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
@@ -159,15 +161,12 @@ public class HCatUtil {
         }
     }
 
-    public static HCatSchema extractSchemaFromStorageDescriptor(
-            StorageDescriptor sd) throws HCatException {
-        if (sd == null) {
-            throw new HCatException(
-                    "Cannot construct partition info from an empty storage descriptor.");
-        }
-        HCatSchema schema = new HCatSchema(HCatUtil.getHCatFieldSchemaList(sd
-                .getCols()));
-        return schema;
+    public static HCatSchema extractSchema(Table table) throws HCatException {
+        return new HCatSchema(HCatUtil.getHCatFieldSchemaList(table.getCols()));
+    }
+
+    public static HCatSchema extractSchema(Partition partition) throws HCatException {
+        return new HCatSchema(HCatUtil.getHCatFieldSchemaList(partition.getCols()));
     }
 
     public static List<FieldSchema> getFieldSchemaList(
@@ -183,14 +182,13 @@ public class HCatUtil {
         }
     }
 
-    public static Table getTable(HiveMetaStoreClient client, String dbName,
-            String tableName) throws Exception {
-        return client.getTable(dbName, tableName);
+    public static Table getTable(HiveMetaStoreClient client, String dbName, String tableName)
+        throws NoSuchObjectException, TException, MetaException {
+      return new Table(client.getTable(dbName, tableName));
     }
 
     public static HCatSchema getTableSchemaWithPtnCols(Table table) throws IOException {
-        HCatSchema tableSchema = new HCatSchema(HCatUtil.getHCatFieldSchemaList(
-                new org.apache.hadoop.hive.ql.metadata.Table(table).getCols()));
+        HCatSchema tableSchema = new HCatSchema(HCatUtil.getHCatFieldSchemaList(table.getCols()));
 
         if (table.getPartitionKeys().size() != 0) {
 
@@ -240,7 +238,7 @@ public class HCatUtil {
             partitionKeyMap.put(field.getName().toLowerCase(), field);
         }
 
-        List<FieldSchema> tableCols = table.getSd().getCols();
+        List<FieldSchema> tableCols = table.getCols();
         List<FieldSchema> newFields = new ArrayList<FieldSchema>();
 
         for (int i = 0; i < partitionSchema.getFields().size(); i++) {

Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java?rev=1379061&r1=1379060&r2=1379061&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java
(original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java
Thu Aug 30 18:53:52 2012
@@ -32,7 +32,7 @@ import org.apache.hadoop.hive.metastore.
 import org.apache.hadoop.hive.metastore.api.MetaException;
 import org.apache.hadoop.hive.metastore.api.Partition;
 import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
-import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.ql.metadata.Table;
 import org.apache.hadoop.mapred.HCatMapRedUtil;
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapreduce.JobContext;
@@ -187,9 +187,8 @@ class FileOutputCommitterContainer exten
         Path src;
         OutputJobInfo jobInfo = HCatOutputFormat.getJobInfo(jobContext);
         if (dynamicPartitioningUsed){
-            src = new Path(getPartitionRootLocation(
-                    jobInfo.getLocation().toString(),jobInfo.getTableInfo().getTable().getPartitionKeysSize()
-            ));
+            src = new Path(getPartitionRootLocation(jobInfo.getLocation(),
+                jobInfo.getTableInfo().getTable().getPartitionKeysSize()));
         }else{
             src = new Path(jobInfo.getLocation());
         }
@@ -243,8 +242,8 @@ class FileOutputCommitterContainer exten
 
         OutputJobInfo jobInfo = HCatOutputFormat.getJobInfo(context);
         Configuration conf = context.getConfiguration();
-        Table table = jobInfo.getTableInfo().getTable();
-        Path tblPath = new Path(table.getSd().getLocation());
+        Table table = new Table(jobInfo.getTableInfo().getTable());
+        Path tblPath = new Path(table.getTTable().getSd().getLocation());
         FileSystem fs = tblPath.getFileSystem(conf);
 
         if( table.getPartitionKeys().size() == 0 ) {
@@ -280,7 +279,8 @@ class FileOutputCommitterContainer exten
             HiveConf hiveConf = HCatUtil.getHiveConf(conf);
             client = HCatUtil.getHiveClient(hiveConf);
 
-            StorerInfo storer = InternalUtil.extractStorerInfo(table.getSd(),table.getParameters());
+            StorerInfo storer =
+                InternalUtil.extractStorerInfo(table.getTTable().getSd(), table.getParameters());
 
             updateTableSchema(client, table, jobInfo.getOutputSchema());
 
@@ -426,12 +426,10 @@ class FileOutputCommitterContainer exten
             Table table, FileSystem fs,
             String grpName, FsPermission perms) throws IOException {
 
-        StorageDescriptor tblSD = table.getSd();
-
         Partition partition = new Partition();
         partition.setDbName(table.getDbName());
         partition.setTableName(table.getTableName());
-        partition.setSd(new StorageDescriptor(tblSD));
+        partition.setSd(new StorageDescriptor(table.getTTable().getSd()));
 
         List<FieldSchema> fields = new ArrayList<FieldSchema>();
         for(HCatFieldSchema fieldSchema : outputSchema.getFields()) {
@@ -500,7 +498,7 @@ class FileOutputCommitterContainer exten
     private String getFinalDynamicPartitionDestination(Table table, Map<String,String>
partKVs) {
         // file:///tmp/hcat_junit_warehouse/employee/_DYN0.7770480401313761/emp_country=IN/emp_state=KA
 ->
         // file:///tmp/hcat_junit_warehouse/employee/emp_country=IN/emp_state=KA
-        Path partPath = new Path(table.getSd().getLocation());
+        Path partPath = new Path(table.getTTable().getSd().getLocation());
         for(FieldSchema partKey : table.getPartitionKeys()){
             partPath = constructPartialPartPath(partPath, partKey.getName().toLowerCase(),
partKVs);
         }
@@ -541,12 +539,12 @@ class FileOutputCommitterContainer exten
         List<FieldSchema> newColumns = HCatUtil.validatePartitionSchema(table, partitionSchema);
 
         if( newColumns.size() != 0 ) {
-            List<FieldSchema> tableColumns = new ArrayList<FieldSchema>(table.getSd().getCols());
+            List<FieldSchema> tableColumns = new ArrayList<FieldSchema>(table.getTTable().getSd().getCols());
             tableColumns.addAll(newColumns);
 
             //Update table schema to add the newly added columns
-            table.getSd().setCols(tableColumns);
-            client.alter_table(table.getDbName(), table.getTableName(), table);
+            table.getTTable().getSd().setCols(tableColumns);
+            client.alter_table(table.getDbName(), table.getTableName(), table.getTTable());
         }
     }
 

Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileOutputFormatContainer.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileOutputFormatContainer.java?rev=1379061&r1=1379060&r2=1379061&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileOutputFormatContainer.java
(original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileOutputFormatContainer.java
Thu Aug 30 18:53:52 2012
@@ -27,7 +27,7 @@ import org.apache.hadoop.hive.metastore.
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
 import org.apache.hadoop.hive.metastore.api.MetaException;
 import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
-import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.ql.metadata.Table;
 import org.apache.hadoop.hive.serde2.SerDe;
 import org.apache.hadoop.io.NullWritable;
 import org.apache.hadoop.io.Writable;
@@ -120,7 +120,7 @@ class FileOutputFormatContainer extends 
             handleDuplicatePublish(context,
                     jobInfo,
                     client,
-                    jobInfo.getTableInfo().getTable());
+                    new Table(jobInfo.getTableInfo().getTable()));
         } catch (MetaException e) {
             throw new IOException(e);
         } catch (TException e) {
@@ -190,7 +190,7 @@ class FileOutputFormatContainer extends 
                     table, outputInfo.getPartitionValues());
             // non-partitioned table
 
-            Path tablePath = new Path(table.getSd().getLocation());
+            Path tablePath = new Path(table.getTTable().getSd().getLocation());
             FileSystem fs = tablePath.getFileSystem(context.getConfiguration());
 
             if ( fs.exists(tablePath) ) {

Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/HCatBaseOutputFormat.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/HCatBaseOutputFormat.java?rev=1379061&r1=1379060&r2=1379061&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/HCatBaseOutputFormat.java
(original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/HCatBaseOutputFormat.java
Thu Aug 30 18:53:52 2012
@@ -25,6 +25,7 @@ import java.util.Map;
 
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.ql.metadata.Table;
 import org.apache.hadoop.io.WritableComparable;
 import org.apache.hadoop.mapreduce.JobContext;
 import org.apache.hadoop.mapreduce.OutputFormat;
@@ -216,7 +217,8 @@ public abstract class HCatBaseOutputForm
       }
     }
     
-    HCatUtil.validatePartitionSchema(jobInfo.getTableInfo().getTable(), schemaWithoutParts);
+    HCatUtil.validatePartitionSchema(
+        new Table(jobInfo.getTableInfo().getTable()), schemaWithoutParts);
     jobInfo.setPosOfPartCols(posOfPartCols);
     jobInfo.setPosOfDynPartCols(posOfDynPartCols);
     jobInfo.setOutputSchema(schemaWithoutParts);

Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/HCatOutputFormat.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/HCatOutputFormat.java?rev=1379061&r1=1379060&r2=1379061&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/HCatOutputFormat.java
(original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/HCatOutputFormat.java
Thu Aug 30 18:53:52 2012
@@ -32,7 +32,7 @@ import org.apache.hadoop.hive.metastore.
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
 import org.apache.hadoop.hive.metastore.api.Index;
 import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
-import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.ql.metadata.Table;
 import org.apache.hadoop.io.WritableComparable;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.OutputCommitter;
@@ -73,7 +73,8 @@ public class HCatOutputFormat extends HC
         Configuration conf = job.getConfiguration();
         HiveConf hiveConf = HCatUtil.getHiveConf(conf);
         client = HCatUtil.getHiveClient(hiveConf);
-        Table table = client.getTable(outputJobInfo.getDatabaseName(), outputJobInfo.getTableName());
+        Table table = HCatUtil.getTable(client, outputJobInfo.getDatabaseName(),
+            outputJobInfo.getTableName());
 
         List<String> indexList = client.listIndexNames(outputJobInfo.getDatabaseName(),
outputJobInfo.getTableName(), Short.MAX_VALUE);
 
@@ -83,7 +84,7 @@ public class HCatOutputFormat extends HC
                 throw new HCatException(ErrorType.ERROR_NOT_SUPPORTED, "Store into a table
with an automatic index from Pig/Mapreduce is not supported");
             }
         }
-        StorageDescriptor sd = table.getSd();
+        StorageDescriptor sd = table.getTTable().getSd();
 
         if (sd.isCompressed()) {
             throw new HCatException(ErrorType.ERROR_NOT_SUPPORTED, "Store into a compressed
partition from Pig/Mapreduce is not supported");
@@ -97,7 +98,7 @@ public class HCatOutputFormat extends HC
             throw new HCatException(ErrorType.ERROR_NOT_SUPPORTED, "Store into a partition
with sorted column definition from Pig/Mapreduce is not supported");
         }
 
-        if (table.getPartitionKeysSize() == 0 ){
+        if (table.getTTable().getPartitionKeysSize() == 0 ){
           if ((outputJobInfo.getPartitionValues() != null) && (!outputJobInfo.getPartitionValues().isEmpty())){
             // attempt made to save partition values in non-partitioned table - throw error.
             throw new HCatException(ErrorType.ERROR_INVALID_PARTITION_VALUES,
@@ -117,7 +118,7 @@ public class HCatOutputFormat extends HC
           }
 
           if ((outputJobInfo.getPartitionValues() == null)
-              || (outputJobInfo.getPartitionValues().size() < table.getPartitionKeysSize())){
+              || (outputJobInfo.getPartitionValues().size() < table.getTTable().getPartitionKeysSize())){
             // dynamic partition usecase - partition values were null, or not all were specified
             // need to figure out which keys are not specified.
             List<String> dynamicPartitioningKeys = new ArrayList<String>();
@@ -128,7 +129,7 @@ public class HCatOutputFormat extends HC
               }
             }
 
-            if (valueMap.size() + dynamicPartitioningKeys.size() != table.getPartitionKeysSize()){
+            if (valueMap.size() + dynamicPartitioningKeys.size() != table.getTTable().getPartitionKeysSize()){
               // If this isn't equal, then bogus key values have been inserted, error out.
               throw new HCatException(ErrorType.ERROR_INVALID_PARTITION_VALUES,"Invalid partition
keys specified");
             }
@@ -148,9 +149,9 @@ public class HCatOutputFormat extends HC
           outputJobInfo.setPartitionValues(valueMap);
         }
 
-        StorageDescriptor tblSD = table.getSd();
-        HCatSchema tableSchema = HCatUtil.extractSchemaFromStorageDescriptor(tblSD);
-        StorerInfo storerInfo = InternalUtil.extractStorerInfo(tblSD,table.getParameters());
+        HCatSchema tableSchema = HCatUtil.extractSchema(table);
+        StorerInfo storerInfo =
+            InternalUtil.extractStorerInfo(table.getTTable().getSd(), table.getParameters());
 
         List<String> partitionCols = new ArrayList<String>();
         for(FieldSchema schema : table.getPartitionKeys()) {
@@ -160,7 +161,7 @@ public class HCatOutputFormat extends HC
        HCatStorageHandler storageHandler = HCatUtil.getStorageHandler(job.getConfiguration(),
storerInfo);
 
         //Serialize the output info into the configuration
-        outputJobInfo.setTableInfo(HCatTableInfo.valueOf(table));
+        outputJobInfo.setTableInfo(HCatTableInfo.valueOf(table.getTTable()));
         outputJobInfo.setOutputSchema(tableSchema);
         harRequested = getHarRequested(hiveConf);
         outputJobInfo.setHarRequested(harRequested);
@@ -169,7 +170,7 @@ public class HCatOutputFormat extends HC
 
         HCatUtil.configureOutputStorageHandler(storageHandler,job,outputJobInfo);
 
-        Path tblPath = new Path(table.getSd().getLocation());
+        Path tblPath = new Path(table.getTTable().getSd().getLocation());
 
         /*  Set the umask in conf such that files/dirs get created with table-dir
          * permissions. Following three assumptions are made:

Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/HCatTableInfo.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/HCatTableInfo.java?rev=1379061&r1=1379060&r2=1379061&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/HCatTableInfo.java (original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/HCatTableInfo.java Thu
Aug 30 18:53:52 2012
@@ -137,17 +137,17 @@ public class HCatTableInfo implements Se
    * @throws IOException
    */
   static HCatTableInfo valueOf(Table table) throws IOException {
-    HCatSchema dataColumns = 
-        HCatUtil.extractSchemaFromStorageDescriptor(table.getSd());
-    StorerInfo storerInfo = 
+    // Explicitly use {@link org.apache.hadoop.hive.ql.metadata.Table} when getting the schema,
+    // but store @{link org.apache.hadoop.hive.metastore.api.Table} as this class is serialized
+    // into the job conf.
+    org.apache.hadoop.hive.ql.metadata.Table mTable =
+        new org.apache.hadoop.hive.ql.metadata.Table(table);
+    HCatSchema schema = HCatUtil.extractSchema(mTable);
+    StorerInfo storerInfo =
         InternalUtil.extractStorerInfo(table.getSd(), table.getParameters());
-    HCatSchema partitionColumns = HCatUtil.getPartitionColumns(table);
-    return new HCatTableInfo(table.getDbName(),
-                             table.getTableName(),
-                             dataColumns,
-                             partitionColumns,
-                             storerInfo,
-                             table);
+    HCatSchema partitionColumns = HCatUtil.getPartitionColumns(mTable);
+    return new HCatTableInfo(table.getDbName(), table.getTableName(), schema,
+        partitionColumns, storerInfo, table);
   }
 
   @Override

Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/InitializeInput.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/InitializeInput.java?rev=1379061&r1=1379060&r2=1379061&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/InitializeInput.java (original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/InitializeInput.java Thu
Aug 30 18:53:52 2012
@@ -30,7 +30,7 @@ import org.apache.hadoop.hive.metastore.
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
 import org.apache.hadoop.hive.metastore.api.Partition;
 import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
-import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.ql.metadata.Table;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hcatalog.common.ErrorType;
 import org.apache.hcatalog.common.HCatConstants;
@@ -94,12 +94,12 @@ public class InitializeInput {
         hiveConf = new HiveConf(HCatInputFormat.class);
       }
       client = HCatUtil.getHiveClient(hiveConf);
-      Table table = client.getTable(inputJobInfo.getDatabaseName(),
-                                    inputJobInfo.getTableName());
+      Table table = HCatUtil.getTable(client, inputJobInfo.getDatabaseName(),
+          inputJobInfo.getTableName());
 
       List<PartInfo> partInfoList = new ArrayList<PartInfo>();
 
-      inputJobInfo.setTableInfo(HCatTableInfo.valueOf(table));
+      inputJobInfo.setTableInfo(HCatTableInfo.valueOf(table.getTTable()));
       if( table.getPartitionKeys().size() != 0 ) {
         //Partitioned table
         List<Partition> parts = client.listPartitionsByFilter(inputJobInfo.getDatabaseName(),
@@ -115,18 +115,19 @@ public class InitializeInput {
 
         // populate partition info
         for (Partition ptn : parts){
-          PartInfo partInfo = extractPartInfo(ptn.getSd(),ptn.getParameters(),
-                                              job.getConfiguration(),
-                                              inputJobInfo);
+          HCatSchema schema = HCatUtil.extractSchema(
+              new org.apache.hadoop.hive.ql.metadata.Partition(table, ptn));
+          PartInfo partInfo = extractPartInfo(schema, ptn.getSd(),
+              ptn.getParameters(), job.getConfiguration(), inputJobInfo);
           partInfo.setPartitionValues(createPtnKeyValueMap(table, ptn));
           partInfoList.add(partInfo);
         }
 
       }else{
         //Non partitioned table
-        PartInfo partInfo = extractPartInfo(table.getSd(),table.getParameters(),
-                                            job.getConfiguration(),
-                                            inputJobInfo);
+        HCatSchema schema = HCatUtil.extractSchema(table);
+        PartInfo partInfo = extractPartInfo(schema, table.getTTable().getSd(),
+            table.getParameters(), job.getConfiguration(), inputJobInfo);
         partInfo.setPartitionValues(new HashMap<String,String>());
         partInfoList.add(partInfo);
       }
@@ -160,29 +161,25 @@ public class InitializeInput {
     return ptnKeyValues;
   }
 
-  static PartInfo extractPartInfo(StorageDescriptor sd,
+  private static PartInfo extractPartInfo(HCatSchema schema, StorageDescriptor sd,
       Map<String,String> parameters, Configuration conf,
       InputJobInfo inputJobInfo) throws IOException{
-    HCatSchema schema = HCatUtil.extractSchemaFromStorageDescriptor(sd);
+
     StorerInfo storerInfo = InternalUtil.extractStorerInfo(sd,parameters);
 
     Properties hcatProperties = new Properties();
-    HCatStorageHandler storageHandler = HCatUtil.getStorageHandler(conf,
-                                                                   storerInfo);
+    HCatStorageHandler storageHandler = HCatUtil.getStorageHandler(conf, storerInfo);
 
     // copy the properties from storageHandler to jobProperties
-    Map<String, String>jobProperties = HCatUtil.getInputJobProperties(
-                                                            storageHandler,
-                                                            inputJobInfo);
+    Map<String, String>jobProperties = HCatUtil.getInputJobProperties(storageHandler,
inputJobInfo);
 
     for (String key : parameters.keySet()){
         hcatProperties.put(key, parameters.get(key));
     }
     // FIXME
     // Bloating partinfo with inputJobInfo is not good
-    return new PartInfo(schema, storageHandler,
-                        sd.getLocation(), hcatProperties,
-                        jobProperties, inputJobInfo.getTableInfo());
+    return new PartInfo(schema, storageHandler, sd.getLocation(),
+        hcatProperties, jobProperties, inputJobInfo.getTableInfo());
   }
 
 }

Modified: incubator/hcatalog/trunk/src/test/org/apache/hcatalog/common/TestHCatUtil.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/org/apache/hcatalog/common/TestHCatUtil.java?rev=1379061&r1=1379060&r2=1379061&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/org/apache/hcatalog/common/TestHCatUtil.java (original)
+++ incubator/hcatalog/trunk/src/test/org/apache/hcatalog/common/TestHCatUtil.java Thu Aug
30 18:53:52 2012
@@ -32,7 +32,7 @@ import org.apache.hadoop.hive.metastore.
 import org.apache.hadoop.hive.metastore.api.Order;
 import org.apache.hadoop.hive.metastore.api.SerDeInfo;
 import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
-import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.ql.metadata.Table;
 import org.apache.hadoop.hive.serde.Constants;
 import org.apache.hcatalog.data.schema.HCatFieldSchema;
 import org.apache.hcatalog.data.schema.HCatSchema;
@@ -120,9 +120,11 @@ public class TestHCatUtil {
         "location", "org.apache.hadoop.mapred.TextInputFormat",
         "org.apache.hadoop.mapred.TextOutputFormat", false, -1, new SerDeInfo(),
         new ArrayList<String>(), new ArrayList<Order>(), new HashMap<String,
String>());
-    Table table = new Table("test_tblname", "test_dbname", "test_owner", 0, 0, 0,
-        sd, new ArrayList<FieldSchema>(), new HashMap<String, String>(),
-        "viewOriginalText", "viewExpandedText", TableType.EXTERNAL_TABLE.name());
+    org.apache.hadoop.hive.metastore.api.Table apiTable =
+        new org.apache.hadoop.hive.metastore.api.Table("test_tblname", "test_dbname", "test_owner",
+            0, 0, 0, sd, new ArrayList<FieldSchema>(), new HashMap<String, String>(),
+            "viewOriginalText", "viewExpandedText", TableType.EXTERNAL_TABLE.name());
+    Table table = new Table(apiTable);
 
     List<HCatFieldSchema> expectedHCatSchema =
         Lists.newArrayList(new HCatFieldSchema("username", HCatFieldSchema.Type.STRING, null));
@@ -133,7 +135,7 @@ public class TestHCatUtil {
     // Add a partition key & ensure its reflected in the schema.
     List<FieldSchema> partitionKeys =
         Lists.newArrayList(new FieldSchema("dt", Constants.STRING_TYPE_NAME, null));
-    table.setPartitionKeys(partitionKeys);
+    table.getTTable().setPartitionKeys(partitionKeys);
     expectedHCatSchema.add(new HCatFieldSchema("dt", HCatFieldSchema.Type.STRING, null));
     Assert.assertEquals(new HCatSchema(expectedHCatSchema),
         HCatUtil.getTableSchemaWithPtnCols(table));
@@ -163,9 +165,11 @@ public class TestHCatUtil {
         false, -1, serDeInfo, new ArrayList<String>(), new ArrayList<Order>(),
         new HashMap<String, String>());
 
-    Table table = new Table("test_tblname", "test_dbname", "test_owner", 0, 0, 0,
-        sd, new ArrayList<FieldSchema>(), new HashMap<String, String>(),
-        "viewOriginalText", "viewExpandedText", TableType.EXTERNAL_TABLE.name());
+    org.apache.hadoop.hive.metastore.api.Table apiTable =
+        new org.apache.hadoop.hive.metastore.api.Table("test_tblname", "test_dbname", "test_owner",
+            0, 0, 0, sd, new ArrayList<FieldSchema>(), new HashMap<String, String>(),
+            "viewOriginalText", "viewExpandedText", TableType.EXTERNAL_TABLE.name());
+    Table table = new Table(apiTable);
 
     List<HCatFieldSchema> expectedHCatSchema = Lists.newArrayList(
         new HCatFieldSchema("myint", HCatFieldSchema.Type.INT, null),



Mime
View raw message