hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From zs...@apache.org
Subject svn commit: r697291 [10/31] - in /hadoop/core/trunk: ./ src/contrib/hive/cli/src/java/org/apache/hadoop/hive/cli/ src/contrib/hive/metastore/if/ src/contrib/hive/metastore/src/gen-javabean/org/apache/hadoop/hive/metastore/api/ src/contrib/hive/metastor...
Date Fri, 19 Sep 2008 23:56:35 GMT
Modified: hadoop/core/trunk/src/contrib/hive/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java?rev=697291&r1=697290&r2=697291&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java (original)
+++ hadoop/core/trunk/src/contrib/hive/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java Fri Sep 19 16:56:30 2008
@@ -41,8 +41,14 @@
 import org.apache.hadoop.hive.metastore.api.SerDeInfo;
 import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
 import org.apache.hadoop.hive.metastore.api.Table;
-import org.apache.hadoop.hive.serde.SerDe;
-import org.apache.hadoop.hive.serde.SerDeUtils;
+import org.apache.hadoop.hive.serde2.Deserializer;
+import org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe;
+import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.serde2.SerDeUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
 import org.apache.hadoop.util.StringUtils;
 
 public class MetaStoreUtils {
@@ -74,14 +80,17 @@
     Table tTable = new Table();
     tTable.setTableName(name);
     tTable.setSd(new StorageDescriptor());
-    tTable.getSd().setSerdeInfo(new SerDeInfo());
-    tTable.getSd().getSerdeInfo().setSerializationLib(org.apache.hadoop.hive.serde.simple_meta.MetadataTypedColumnsetSerDe.class.getName());
-    tTable.getSd().getSerdeInfo().setSerializationFormat("1");
+    StorageDescriptor sd = tTable.getSd();
+    sd.setSerdeInfo(new SerDeInfo());
+    SerDeInfo serdeInfo = sd.getSerdeInfo();
+    serdeInfo.setSerializationLib(MetadataTypedColumnsetSerDe.class.getName());
+    serdeInfo.setParameters(new HashMap<String, String>());
+    serdeInfo.getParameters().put(org.apache.hadoop.hive.serde.Constants.SERIALIZATION_FORMAT, "1");
     
     List<FieldSchema>  fields = new ArrayList<FieldSchema>();
-    tTable.getSd().setCols(fields);
+    sd.setCols(fields);
     for (String col: columns) {
-      FieldSchema field = new FieldSchema(col, Constants.STRING_TYPE_NAME, "default string type");
+      FieldSchema field = new FieldSchema(col, org.apache.hadoop.hive.serde.Constants.STRING_TYPE_NAME, "'default'");
       fields.add(field);
     }
 
@@ -89,12 +98,12 @@
     for (String partCol : partCols) {
       FieldSchema part = new FieldSchema();
       part.setName(partCol);
-      part.setType(Constants.STRING_TYPE_NAME); // default partition key
+      part.setType(org.apache.hadoop.hive.serde.Constants.STRING_TYPE_NAME); // default partition key
       tTable.getPartitionKeys().add(part);
     }
     // not sure why these are needed
-    tTable.getSd().getSerdeInfo().setSerializationLib(org.apache.hadoop.hive.serde.simple_meta.MetadataTypedColumnsetSerDe.shortName());
-    tTable.getSd().setNumBuckets(-1);
+    serdeInfo.setSerializationLib(MetadataTypedColumnsetSerDe.shortName());
+    sd.setNumBuckets(-1);
     return tTable;
   }
 
@@ -122,25 +131,25 @@
 
 
   /**
-   * getSerDe
+   * getDeserializer
    *
-   * Get the SerDe for a table given its name and properties.
+   * Get the Deserializer for a table given its name and properties.
    *
    * @param name the name of the table
    * @param conf - hadoop config
    * @param p - the properties to use to instantiate the schema
-   * @return the SerDe
-   * @exception MetaException if any problems instantiating the serde
+   * @return the Deserializer
+   * @exception MetaException if any problems instantiating the Deserializer
    *
    * todo - this should move somewhere into serde.jar
    *
    */
-  static public SerDe getSerDe(Configuration conf, Properties schema) throws MetaException  {
+  static public Deserializer getDeserializer(Configuration conf, Properties schema) throws MetaException  {
     String lib = schema.getProperty(org.apache.hadoop.hive.serde.Constants.SERIALIZATION_LIB);
     try {
-      SerDe serDe = SerDeUtils.lookupSerDe(lib);
-      ((SerDe)serDe).initialize(conf, schema);
-      return serDe;
+      Deserializer deserializer = SerDeUtils.lookupDeserializer(lib);
+      ((Deserializer)deserializer).initialize(conf, schema);
+      return deserializer;
     } catch (Exception e) {
       LOG.error("error in initSerDe: " + e.getClass().getName() + " " + e.getMessage());
       MetaStoreUtils.printStackTrace(e);
@@ -149,25 +158,25 @@
   }
 
   /**
-   * getSerDe
+   * getDeserializer
    *
-   * Get the SerDe for a table given its name and properties.
+   * Get the Deserializer for a table given its name and properties.
    *
    * @param name the name of the table
    * @param conf - hadoop config
    * @param p - SerDe info
-   * @return the SerDe
-   * @exception MetaException if any problems instantiating the serde
+   * @return the Deserializer
+   * @exception MetaException if any problems instantiating the Deserializer
    *
    * todo - this should move somewhere into serde.jar
    *
    */
-  static public SerDe getSerDe(Configuration conf, org.apache.hadoop.hive.metastore.api.Table table) throws MetaException  {
+  static public Deserializer getDeserializer(Configuration conf, org.apache.hadoop.hive.metastore.api.Table table) throws MetaException  {
     String lib = table.getSd().getSerdeInfo().getSerializationLib();
     try {
-      SerDe serDe = SerDeUtils.lookupSerDe(lib);
-      ((SerDe)serDe).initialize(conf, MetaStoreUtils.getSchema(table));
-      return serDe;
+      Deserializer deserializer = SerDeUtils.lookupDeserializer(lib);
+      deserializer.initialize(conf, MetaStoreUtils.getSchema(table));
+      return deserializer;
     } catch (Exception e) {
       LOG.error("error in initSerDe: " + e.getClass().getName() + " " + e.getMessage());
       MetaStoreUtils.printStackTrace(e);
@@ -220,9 +229,9 @@
 
 
   /**
-   * validateTableName
+   * validateName
    *
-   * Checks the name conforms to our standars which are: "[a-zA-z-_.0-9]+".
+   * Checks the name conforms to our standars which are: "[a-zA-z-_0-9]+".
    * checks this is just characters and numbers and _ and . and -
    *
    * @param tableName the name to validate
@@ -249,19 +258,22 @@
     for(Enumeration<?> e = p.propertyNames(); e.hasMoreElements() ; ) {
       String key = (String)e.nextElement();
       String oldName = p.getProperty(key);
-
       oldName = oldName.replace("com.facebook.infrastructure.tprofiles","com.facebook.serde.tprofiles");
-
+      
       oldName = oldName.replace("com.facebook.infrastructure.hive_context","com.facebook.serde.hive_context");
+      oldName = oldName.replace("com.facebook.serde.hive_context","com.facebook.serde2.hive_context");
 
-      oldName = oldName.replace("com.facebook.thrift.hive.MetadataTypedColumnsetSerDe",org.apache.hadoop.hive.serde.simple_meta.MetadataTypedColumnsetSerDe.class.getName());
+      oldName = oldName.replace("com.facebook.thrift.hive.MetadataTypedColumnsetSerDe",org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe.class.getName());
 
       // columnset serde
-      oldName = oldName.replace("com.facebook.thrift.hive.columnsetSerDe",org.apache.hadoop.hive.serde.thrift.columnsetSerDe.class.getName());
+      oldName = oldName.replace("com.facebook.thrift.hive.columnsetSerDe",org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe.class.getName());
+      oldName = oldName.replace("org.apache.hadoop.hive.serde.simple_meta.MetadataTypedColumnsetSerDe",
+          org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe.class.getName());
 
       // thrift serde
-      oldName = oldName.replace("com.facebook.thrift.hive.ThriftHiveSerDe",org.apache.hadoop.hive.serde.thrift.ThriftSerDe.class.getName());
-
+      oldName = oldName.replace("com.facebook.thrift.hive.ThriftHiveSerDe",org.apache.hadoop.hive.serde2.ThriftDeserializer.class.getName());
+      oldName = oldName.replace("org.apache.hadoop.hive.serde.thrift.ThriftSerDe",
+          org.apache.hadoop.hive.serde2.ThriftDeserializer.class.getName());
       p.setProperty(key,oldName);
     }
     return p;
@@ -275,7 +287,7 @@
     return "map<" + k +"," + v + ">";
   }
 
-  public static Table getTable(Properties schema) {
+  public static Table getTable(Configuration conf, Properties schema) throws MetaException {
     Table t = new Table();
     t.setSd(new StorageDescriptor());
     t.setTableName(schema.getProperty(org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_NAME));
@@ -285,7 +297,7 @@
     t.getSd().setOutputFormat(schema.getProperty(org.apache.hadoop.hive.metastore.api.Constants.FILE_OUTPUT_FORMAT,
           org.apache.hadoop.mapred.SequenceFileOutputFormat.class.getName())); 
     t.setPartitionKeys(new ArrayList<FieldSchema>());
-    t.setDatabase(MetaStoreUtils.DEFAULT_DATABASE_NAME);
+    t.setDbName(MetaStoreUtils.DEFAULT_DATABASE_NAME);
     String part_cols_str = schema.getProperty(org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_PARTITION_COLUMNS);
     t.setPartitionKeys(new ArrayList<FieldSchema>());
     if (part_cols_str != null && (part_cols_str.trim().length() != 0)) {
@@ -293,7 +305,7 @@
       for (String key: part_keys) {
         FieldSchema part = new FieldSchema();
         part.setName(key);
-        part.setType(Constants.STRING_TYPE_NAME); // default partition key
+        part.setType(org.apache.hadoop.hive.serde.Constants.STRING_TYPE_NAME); // default partition key
         t.getPartitionKeys().add(part);
       }
     }
@@ -306,28 +318,37 @@
     }
     
     t.getSd().setSerdeInfo(new SerDeInfo());
+    t.getSd().getSerdeInfo().setParameters(new HashMap<String, String>());
     t.getSd().getSerdeInfo().setName(t.getTableName());
-    t.getSd().getSerdeInfo().setSerializationClass(schema.getProperty(org.apache.hadoop.hive.serde.Constants.SERIALIZATION_CLASS)); 
-    t.getSd().getSerdeInfo().setSerializationFormat(schema.getProperty(org.apache.hadoop.hive.serde.Constants.SERIALIZATION_FORMAT)); 
     t.getSd().getSerdeInfo().setSerializationLib(schema.getProperty(org.apache.hadoop.hive.serde.Constants.SERIALIZATION_LIB));
-    if(t.getSd().getSerdeInfo().getSerializationClass() == null || (t.getSd().getSerdeInfo().getSerializationClass().length() == 0)) {
-      t.getSd().getSerdeInfo().setSerializationClass(schema.getProperty(org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_SERDE));
+    setSerdeParam(t.getSd().getSerdeInfo(), schema, org.apache.hadoop.hive.serde.Constants.SERIALIZATION_CLASS);
+    setSerdeParam(t.getSd().getSerdeInfo(), schema, org.apache.hadoop.hive.serde.Constants.SERIALIZATION_FORMAT);
+    if(org.apache.commons.lang.StringUtils.isNotBlank(schema.getProperty(org.apache.hadoop.hive.serde.Constants.SERIALIZATION_CLASS))) {
+      setSerdeParam(t.getSd().getSerdeInfo(), schema, org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_SERDE);
     }
+    // needed for MetadataTypedColumnSetSerDe
+    setSerdeParam(t.getSd().getSerdeInfo(), schema, org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_COLUMNS);
     
     String colstr = schema.getProperty(org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_COLUMNS);
     List<FieldSchema>  fields = new ArrayList<FieldSchema>();
-    t.getSd().setCols(fields);
     if(colstr != null) {
       String[] cols =  colstr.split(",");
       for (String colName : cols) {
-        FieldSchema col = new FieldSchema(colName, Constants.STRING_TYPE_NAME, "default string type");
+        FieldSchema col = new FieldSchema(colName, org.apache.hadoop.hive.serde.Constants.STRING_TYPE_NAME, "'default'");
         fields.add(col);
       }
     } 
     
     if(fields.size() == 0) {
-      fields.add(new FieldSchema("__SERDE__", t.getSd().getSerdeInfo().getSerializationLib(), ""));
+      // get the fields from serde
+      try {
+        fields = getFieldsFromDeserializer(t.getTableName(), getDeserializer(conf, schema));
+      } catch (SerDeException e) {
+        LOG.error(StringUtils.stringifyException(e));
+        throw new MetaException("Invalid serde or schema. " + e.getMessage());
+      }
     }
+    t.getSd().setCols(fields);
     
     // remove all the used up parameters to find out the remaining parameters
     schema.remove(Constants.META_TABLE_NAME);
@@ -337,9 +358,9 @@
     schema.remove(Constants.META_TABLE_PARTITION_COLUMNS);
     schema.remove(Constants.BUCKET_COUNT);
     schema.remove(Constants.BUCKET_FIELD_NAME);
-    schema.remove(Constants.SERIALIZATION_CLASS);
-    schema.remove(Constants.SERIALIZATION_FORMAT);
-    schema.remove(Constants.SERIALIZATION_LIB);
+    schema.remove(org.apache.hadoop.hive.serde.Constants.SERIALIZATION_CLASS);
+    schema.remove(org.apache.hadoop.hive.serde.Constants.SERIALIZATION_FORMAT);
+    schema.remove(org.apache.hadoop.hive.serde.Constants.SERIALIZATION_LIB);
     schema.remove(Constants.META_TABLE_SERDE);
     schema.remove(Constants.META_TABLE_COLUMNS);
     
@@ -352,6 +373,13 @@
     return t;
   }
 
+  private static void setSerdeParam(SerDeInfo sdi, Properties schema, String param) {
+    String val = schema.getProperty(param);
+    if(org.apache.commons.lang.StringUtils.isNotBlank(val)) {
+      sdi.getParameters().put(param, val);
+    }
+  }
+
   public static Properties getSchema(org.apache.hadoop.hive.metastore.api.Table tbl) {
     Properties schema = new Properties();
     String inputFormat = tbl.getSd().getInputFormat();
@@ -372,12 +400,7 @@
     if (tbl.getSd().getBucketCols().size() > 0) {
       schema.setProperty(org.apache.hadoop.hive.metastore.api.Constants.BUCKET_FIELD_NAME, tbl.getSd().getBucketCols().get(0));
     }
-    if(tbl.getSd().getSerdeInfo().getSerializationClass() != null) {
-      schema.setProperty(org.apache.hadoop.hive.serde.Constants.SERIALIZATION_CLASS, tbl.getSd().getSerdeInfo().getSerializationClass());
-    }
-    if(tbl.getSd().getSerdeInfo().getSerializationFormat() != null) {
-      schema.setProperty(org.apache.hadoop.hive.serde.Constants.SERIALIZATION_FORMAT, tbl.getSd().getSerdeInfo().getSerializationFormat());
-    }
+    schema.putAll(tbl.getSd().getSerdeInfo().getParameters());
     if(tbl.getSd().getSerdeInfo().getSerializationLib() != null) {
       schema.setProperty(org.apache.hadoop.hive.serde.Constants.SERIALIZATION_LIB, tbl.getSd().getSerdeInfo().getSerializationLib());
     }
@@ -440,4 +463,46 @@
     LOG.error(StringUtils.stringifyException(e));
     throw new MetaException("Got exception: " + e.getClass().getName() + " " + e.getMessage());
   }
+
+  /**
+   * @param tableName
+   * @param deserializer
+   * @return
+   * @throws SerDeException
+   * @throws MetaException
+   */
+  static List<FieldSchema> getFieldsFromDeserializer(String tableName, Deserializer deserializer) throws SerDeException, MetaException {
+    ObjectInspector oi = deserializer.getObjectInspector();
+    String [] names = tableName.split("\\.");
+    String last_name = names[names.length-1];
+    for(int i = 1; i < names.length; i++) {
+      if (!(oi instanceof StructObjectInspector)) {
+        oi = deserializer.getObjectInspector();
+        break;
+      }
+      StructObjectInspector soi = (StructObjectInspector)oi;
+      StructField sf = soi.getStructFieldRef(names[i]);
+      if (sf == null) {
+        // If invalid field, then return the schema of the table
+        oi = deserializer.getObjectInspector();
+        break;
+      } else {
+        oi = sf.getFieldObjectInspector();
+      }
+    }
+
+    ArrayList<FieldSchema> str_fields = new ArrayList<FieldSchema>(); 
+    // rules on how to recurse the ObjectInspector based on its type
+    if (oi.getCategory() != Category.STRUCT) {
+      str_fields.add(new FieldSchema(last_name, oi.getTypeName(), "from deserializer"));
+    } else {
+      List<? extends StructField> fields = ((StructObjectInspector)oi).getAllStructFieldRefs();
+      for(int i=0; i<fields.size(); i++) {
+        String fieldName = fields.get(i).getFieldName();
+        String fieldTypeName = fields.get(i).getFieldObjectInspector().getTypeName();
+        str_fields.add(new FieldSchema(fieldName, fieldTypeName, "from deserializer"));
+      }
+    }
+    return str_fields;
+  }
 }

Modified: hadoop/core/trunk/src/contrib/hive/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java?rev=697291&r1=697290&r2=697291&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java (original)
+++ hadoop/core/trunk/src/contrib/hive/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java Fri Sep 19 16:56:30 2008
@@ -25,6 +25,7 @@
 import java.util.Collection;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
 import java.util.Properties;
 import java.util.Map.Entry;
 
@@ -140,28 +141,31 @@
     URL url= classLoader.getResource(JPOX_CONFIG);
     prop = new Properties();
     if (url == null) {
-      LOG.info(JPOX_CONFIG + " not found");
-      //throw new RuntimeException("Properties file not found " + JPOX_CONFIG);
+      LOG.info(JPOX_CONFIG + " not found.");
     } else {
       LOG.info("found resource " + JPOX_CONFIG + " at " + url);
       try {
         InputStream is = url.openStream();
         if (is == null) {
-          throw new RuntimeException("Properties file not found " + JPOX_CONFIG);
+          throw new RuntimeException("Properties file " + url + " couldn't be opened");
         }
-
         prop.load(is);
       } catch (IOException ex) {
         throw new RuntimeException("could not load: " + JPOX_CONFIG, ex);
       }
     }
-    addHiveConfProperty(javax.jdo.Constants.PROPERTY_CONNECTION_URL);
-    addHiveConfProperty(javax.jdo.Constants.PROPERTY_CONNECTION_DRIVER_NAME);
-    addHiveConfProperty(javax.jdo.Constants.PROPERTY_CONNECTION_USER_NAME);
-    addHiveConfProperty(javax.jdo.Constants.PROPERTY_CONNECTION_PASSWORD);
-    addHiveConfProperty(javax.jdo.Constants.PROPERTY_NONTRANSACTIONAL_READ);
-    addHiveConfProperty(javax.jdo.Constants.PROPERTY_DETACH_ALL_ON_COMMIT);
-    addHiveConfProperty(javax.jdo.Constants.PROPERTY_PERSISTENCE_MANAGER_FACTORY_CLASS);
+    
+    Iterator<Map.Entry<String, String>> iter = hiveConf.iterator();
+    while(iter.hasNext()) {
+      Map.Entry<String, String> e = iter.next();
+      if(e.getKey().contains("jpox") || e.getKey().contains("jdo")) {
+        Object prevVal = prop.setProperty(e.getKey(), e.getValue());
+        if(LOG.isDebugEnabled()) {
+          LOG.debug("Overriding " + e.getKey() + " value " + prevVal 
+              + " from  jpox.properties with " + e.getValue());
+        }
+      }
+    }
 
     if(LOG.isDebugEnabled()) {
       for (Entry<Object, Object> e: prop.entrySet()) {
@@ -170,15 +174,6 @@
     }
   }
 
-  /**
-   * Overwrites the default jpox.properties attribute with what is specified in hive-default.xml
-   */
-  private void addHiveConfProperty(String param) {
-    String val = null;
-    if( (val=hiveConf.get(param)) != null) {
-      prop.setProperty(param, val);
-    }
-  }
   private static PersistenceManagerFactory getPMF() {
     if(pmf == null) {
       pmf = JDOHelper.getPersistenceManagerFactory(prop);
@@ -232,7 +227,7 @@
    */
   @SuppressWarnings("nls")
   public boolean commitTransaction() {
-    assert(this.openTrasactionCalls > 1);
+    assert(this.openTrasactionCalls >= 1);
     if(!currentTransaction.isActive()) {
       throw new RuntimeException("Commit is called, but transaction is not active. Either there are" +
           "mismatching open and close calls or rollback was called in the same trasaction");
@@ -259,8 +254,10 @@
    * Rolls back the current transaction if it is active
    */
   public void rollbackTransaction() {
-    assert(this.openTrasactionCalls > 1);
-    this.openTrasactionCalls--;
+    if(this.openTrasactionCalls < 1) {
+      return;
+    }
+    this.openTrasactionCalls = 0;
     if(currentTransaction.isActive() && transactionStatus != TXN_STATUS.ROLLBACK) {
       transactionStatus = TXN_STATUS.ROLLBACK;
        // could already be rolled back
@@ -271,7 +268,7 @@
   public boolean createDatabase(Database db) {
     boolean success = false;
     boolean commited = false;
-    MDatabase mdb = new MDatabase(db.getName().toLowerCase(), db.getLocationUri());
+    MDatabase mdb = new MDatabase(db.getName().toLowerCase(), db.getDescription());
     try {
       openTransaction();
       pm.makePersistent(mdb);
@@ -326,7 +323,7 @@
         rollbackTransaction();
       }
     }
-    return new Database(db.getName(), db.getLocation());
+    return new Database(db.getName(), db.getDescription());
   }
 
   public boolean dropDatabase(String dbname) {
@@ -587,10 +584,10 @@
     if(tbl == null) return null;
     MDatabase mdb = null;
     try {
-      mdb = this.getMDatabase(tbl.getDatabase());
+      mdb = this.getMDatabase(tbl.getDbName());
     } catch (NoSuchObjectException e) {
       LOG.error(StringUtils.stringifyException(e));
-      throw new InvalidObjectException("Database " + tbl.getDatabase() + " doesn't exsit.");
+      throw new InvalidObjectException("Database " + tbl.getDbName() + " doesn't exsit.");
     }
     return new MTable(tbl.getTableName().toLowerCase(),
         mdb,
@@ -650,26 +647,14 @@
   private SerDeInfo converToSerDeInfo(MSerDeInfo ms) throws MetaException {
    if(ms == null) throw new MetaException("Invalid SerDeInfo object");
    return new SerDeInfo(ms.getName(),
-       ms.getSerializationFormat(),
-       ms.getSerializationClass(),
        ms.getSerializationLib(),
-       ms.getFieldDelim(),
-       ms.getCollectionItemDelim(),
-       ms.getMapKeyDelim(),
-       ms.getLineDelim(),
        ms.getParameters()); 
   }
   
   private MSerDeInfo converToMSerDeInfo(SerDeInfo ms) throws MetaException {
     if(ms == null) throw new MetaException("Invalid SerDeInfo object");
     return new MSerDeInfo(ms.getName(),
-        ms.getSerializationFormat(),
-        ms.getSerializationClass(),
         ms.getSerializationLib(),
-        ms.getFieldDelim(),
-        ms.getCollectionItemDelim(),
-        ms.getMapKeyDelim(),
-        ms.getLineDelim(),
         ms.getParameters()); 
    }
   
@@ -697,7 +682,7 @@
         sd.getLocation(),
         sd.getInputFormat(),
         sd.getOutputFormat(),
-        sd.isIsCompressed(),
+        sd.isCompressed(),
         sd.getNumBuckets(),
         converToMSerDeInfo(sd.getSerdeInfo()),
         sd.getBucketCols(),
@@ -762,7 +747,7 @@
     if(part == null) {
       return null;
     }
-    MTable mt = getMTable(part.getDatabase(), part.getTableName());
+    MTable mt = getMTable(part.getDbName(), part.getTableName());
     if(mt == null) {
       throw new InvalidObjectException("Partition doesn't have a valid table or database name");
     }
@@ -822,6 +807,34 @@
     return parts;
   }
 
+
+  //TODO:pc implement max
+  public List<String> listPartitionNames(String dbName, String tableName, short max) throws MetaException {
+    List<String> pns = new ArrayList<String>();
+    boolean success = false;
+    try {
+      openTransaction();
+      LOG.debug("Executing getPartitionNames");
+      dbName = dbName.toLowerCase();
+      tableName = tableName.toLowerCase();
+      Query q = pm.newQuery("select partitionName from org.apache.hadoop.hive.metastore.model.MPartition where table.database.name == t1 && table.tableName == t2");
+      q.declareParameters("java.lang.String t1, java.lang.String t2");
+      q.setResult("partitionName");
+      Collection names = (Collection) q.execute(dbName.trim(), tableName.trim());
+      pns = new ArrayList<String>(); 
+      for (Iterator i = names.iterator (); i.hasNext ();) {
+          pns.add((String) i.next ()); 
+      }
+      success = commitTransaction();
+    } finally {
+      if(!success) {
+        rollbackTransaction();
+        success = false;
+      }
+    }
+    return pns;
+  }
+  
   // TODO:pc implement max
   private List<MPartition> listMPartitions(String dbName, String tableName, int max) {
     boolean success = false;
@@ -853,11 +866,11 @@
       openTransaction();
       name = name.toLowerCase();
       dbname = dbname.toLowerCase();
-      MTable newt = this.getMTable(newTable.getDatabase(), newTable.getTableName());
+      MTable newt = this.getMTable(newTable.getDbName(), newTable.getTableName());
       if(newt != null) {
-        if(!newTable.getTableName().equals(name) || !newTable.getDatabase().equals(dbname)) {
+        if(!newTable.getTableName().equals(name) || !newTable.getDbName().equals(dbname)) {
           // if the old table and new table aren't the same
-          throw new InvalidObjectException("new table " + newTable.getDatabase() +" already exists");
+          throw new InvalidObjectException("new table " + newTable.getDbName() +" already exists");
         }
       }
       newt = convertToMTable(newTable);

Modified: hadoop/core/trunk/src/contrib/hive/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java?rev=697291&r1=697290&r2=697291&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java (original)
+++ hadoop/core/trunk/src/contrib/hive/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java Fri Sep 19 16:56:30 2008
@@ -90,4 +90,6 @@
       throws InvalidObjectException, MetaException;
 
   public List<String> getTables(String dbName, String pattern) throws MetaException;
+
+  public abstract List<String> listPartitionNames(String db_name, String tbl_name, short max_parts) throws MetaException;
 }

Modified: hadoop/core/trunk/src/contrib/hive/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java?rev=697291&r1=697290&r2=697291&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java (original)
+++ hadoop/core/trunk/src/contrib/hive/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java Fri Sep 19 16:56:30 2008
@@ -144,7 +144,7 @@
 
   public static String makePartName(List<FieldSchema> partCols, List<String> vals) throws MetaException {
     if ((partCols.size() != vals.size()) || (partCols.size() == 0)) {
-      throw new MetaException("Invalid key & values");
+      throw new MetaException("Invalid partition key & values");
     }
     StringBuilder name = new StringBuilder();
     for(int i=0; i< partCols.size(); i++) {

Modified: hadoop/core/trunk/src/contrib/hive/metastore/src/model/org/apache/hadoop/hive/metastore/model/MDatabase.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/metastore/src/model/org/apache/hadoop/hive/metastore/model/MDatabase.java?rev=697291&r1=697290&r2=697291&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/metastore/src/model/org/apache/hadoop/hive/metastore/model/MDatabase.java (original)
+++ hadoop/core/trunk/src/contrib/hive/metastore/src/model/org/apache/hadoop/hive/metastore/model/MDatabase.java Fri Sep 19 16:56:30 2008
@@ -27,7 +27,7 @@
  */
 public class MDatabase {
   private String name;
-  private String location;
+  private String description;
 
   /**
    * Default construction to keep jpox/jdo happy
@@ -41,7 +41,7 @@
    */
   public MDatabase(String name, String location) {
     this.name = name;
-    this.location = location;
+    this.description = location;
   }
 
   /**
@@ -59,16 +59,17 @@
   }
 
   /**
-   * @return the location
+   * @return the description
    */
-  public String getLocation() {
-    return location;
+  public String getDescription() {
+    return description;
   }
 
   /**
-   * @param location the location to set
+   * @param description the description to set
    */
-  public void setLocation(String location) {
-    this.location = location;
+  public void setDescription(String description) {
+    this.description = description;
   }
+
 }

Modified: hadoop/core/trunk/src/contrib/hive/metastore/src/model/org/apache/hadoop/hive/metastore/model/MSerDeInfo.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/metastore/src/model/org/apache/hadoop/hive/metastore/model/MSerDeInfo.java?rev=697291&r1=697290&r2=697291&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/metastore/src/model/org/apache/hadoop/hive/metastore/model/MSerDeInfo.java (original)
+++ hadoop/core/trunk/src/contrib/hive/metastore/src/model/org/apache/hadoop/hive/metastore/model/MSerDeInfo.java Fri Sep 19 16:56:30 2008
@@ -22,69 +22,21 @@
 
 public class MSerDeInfo {
   private String name;
-  private String serializationFormat;
-  private String serializationClass;
   private String serializationLib;
-  private String fieldDelim;
-  private String collectionItemDelim;
-  private String mapKeyDelim;
-  private String lineDelim;
   private Map<String, String> parameters;
 
   /**
    * @param name
-   * @param serializationFormat
-   * @param serializationClass
    * @param serializationLib
-   * @param fieldDelim
-   * @param collectionItemDelim
-   * @param mapKeyDelim
-   * @param lineDelim
    * @param parameters
    */
-  public MSerDeInfo(String name, String serializationFormat, String serializationClass,
-      String serializationLib, String fieldDelim, String collectionItemDelim, String mapKeyDelim,
-      String lineDelim, Map<String, String> parameters) {
+  public MSerDeInfo(String name, String serializationLib, Map<String, String> parameters) {
     this.name = name;
-    this.serializationFormat = serializationFormat;
-    this.serializationClass = serializationClass;
     this.serializationLib = serializationLib;
-    this.fieldDelim = fieldDelim;
-    this.collectionItemDelim = collectionItemDelim;
-    this.mapKeyDelim = mapKeyDelim;
-    this.lineDelim = lineDelim;
     this.parameters = parameters;
   }
 
   /**
-   * @return the serializationFormat
-   */
-  public String getSerializationFormat() {
-    return serializationFormat;
-  }
-
-  /**
-   * @param serializationFormat the serializationFormat to set
-   */
-  public void setSerializationFormat(String serializationFormat) {
-    this.serializationFormat = serializationFormat;
-  }
-
-  /**
-   * @return the serializationClass
-   */
-  public String getSerializationClass() {
-    return serializationClass;
-  }
-
-  /**
-   * @param serializationClass the serializationClass to set
-   */
-  public void setSerializationClass(String serializationClass) {
-    this.serializationClass = serializationClass;
-  }
-
-  /**
    * @return the serializationLib
    */
   public String getSerializationLib() {
@@ -113,62 +65,6 @@
   }
 
   /**
-   * @return the fieldDelim
-   */
-  public String getFieldDelim() {
-    return fieldDelim;
-  }
-
-  /**
-   * @param fieldDelim the fieldDelim to set
-   */
-  public void setFieldDelim(String fieldDelim) {
-    this.fieldDelim = fieldDelim;
-  }
-
-  /**
-   * @return the collectionItemDelim
-   */
-  public String getCollectionItemDelim() {
-    return collectionItemDelim;
-  }
-
-  /**
-   * @param collectionItemDelim the collectionItemDelim to set
-   */
-  public void setCollectionItemDelim(String collectionItemDelim) {
-    this.collectionItemDelim = collectionItemDelim;
-  }
-
-  /**
-   * @return the mapKeyDelim
-   */
-  public String getMapKeyDelim() {
-    return mapKeyDelim;
-  }
-
-  /**
-   * @param mapKeyDelim the mapKeyDelim to set
-   */
-  public void setMapKeyDelim(String mapKeyDelim) {
-    this.mapKeyDelim = mapKeyDelim;
-  }
-
-  /**
-   * @return the lineDelim
-   */
-  public String getLineDelim() {
-    return lineDelim;
-  }
-
-  /**
-   * @param lineDelim the lineDelim to set
-   */
-  public void setLineDelim(String lineDelim) {
-    this.lineDelim = lineDelim;
-  }
-
-  /**
    * @return the parameters
    */
   public Map<String, String> getParameters() {

Modified: hadoop/core/trunk/src/contrib/hive/metastore/src/model/package.jdo
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/metastore/src/model/package.jdo?rev=697291&r1=697290&r2=697291&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/metastore/src/model/package.jdo (original)
+++ hadoop/core/trunk/src/contrib/hive/metastore/src/model/package.jdo Fri Sep 19 16:56:30 2008
@@ -4,12 +4,15 @@
 <jdo>  
   <package name="org.apache.hadoop.hive.metastore.model">  
     <class name="MDatabase" identity-type="datastore" table="DBS" detachable="true">  
+      <datastore-identity>
+        <column name="DB_ID"/>
+      </datastore-identity>
       <field name="name">  
         <column name="NAME" length="128" jdbc-type="VARCHAR"/>  
         <index name="UniqueDatabase" unique="true"/>
       </field>
-      <field name="location">  
-        <column name="location" length="256" jdbc-type="VARCHAR"/>  
+      <field name="description">  
+        <column name="DESC" length="767" jdbc-type="VARCHAR"/>  
       </field>  
     </class>
 
@@ -18,10 +21,10 @@
         <column name="FNAME" length="128" jdbc-type="VARCHAR"/>
       </field>
       <field name="type" >
-        <column name="FTYPE" length="1024" jdbc-type="VARCHAR" allows-null="false"/>
+        <column name="FTYPE" length="767" jdbc-type="VARCHAR" allows-null="false"/>
       </field>
       <field name="comment" >
-        <column name="FCOMMENT" length="1024" jdbc-type="VARCHAR" allows-null="true"/>
+        <column name="FCOMMENT" length="767" jdbc-type="VARCHAR" allows-null="true"/>
       </field>
     </class>
 
@@ -59,24 +62,27 @@
     </class>
 
     <class name="MTable" table="TBLS" identity-type="datastore" detachable="true">
+      <datastore-identity>
+        <column name="TBL_ID"/>
+      </datastore-identity>
       <index name="UniqueTable" unique="true">
         <column name="TBL_NAME"/>
-        <column name="DATABASE_ID"/>
+        <column name="DB_ID"/>
       </index>
       <field name="tableName">
         <column name="TBL_NAME" length="128" jdbc-type="VARCHAR"/>
       </field>
       <field name="database">
-        <column name="DATABASE_ID"/>
+        <column name="DB_ID"/>
       </field>
       <field name="partitionKeys" table="PARTITION_KEYS" >
         <collection element-type="MFieldSchema"/>
         <join>
           <primary-key name="PARTITION_KEY_PK">
-            <column name="TABLE_ID"/>
+            <column name="TBL_ID"/>
             <column name="PKEY_NAME"/>
           </primary-key>
-          <column name="TABLE_ID"/>
+          <column name="TBL_ID"/>
         </join>
         <element>
           <embedded>
@@ -84,15 +90,17 @@
               <column name="PKEY_NAME" length="128" jdbc-type="VARCHAR"/>
               </field>
             <field name="type">
-              <column name="PKEY_TYPE" length="1024" jdbc-type="VARCHAR" allows-null="false"/>
+              <column name="PKEY_TYPE" length="767" jdbc-type="VARCHAR" allows-null="false"/>
             </field>
             <field name="comment" >
-              <column name="PKEY_COMMENT" length="1024" jdbc-type="VARCHAR" allows-null="true"/>
+              <column name="PKEY_COMMENT" length="767" jdbc-type="VARCHAR" allows-null="true"/>
             </field>
           </embedded>
         </element>
       </field>
-      <field name="sd" dependent="true"/>
+      <field name="sd" dependent="true">
+        <column name="SD_ID"/>
+      </field>
       <field name="owner">
         <column name="OWNER" length="128" jdbc-type="VARCHAR"/>
       </field>
@@ -107,53 +115,55 @@
       </field>
       <field name="parameters" table="TABLE_PARAMS">
         <map key-type="java.lang.String" value-type="java.lang.String"/>
-        <join/>
+        <join>
+          <column name="TBL_ID"/>
+        </join>
         <key>
            <column name="PARAM_KEY" length="256" jdbc-type="VARCHAR"/>
         </key>
         <value>
-           <column name="PARAM_VALUE" length="1024" jdbc-type="VARCHAR"/>
+           <column name="PARAM_VALUE" length="767" jdbc-type="VARCHAR"/>
         </value>
       </field>
     </class>
 
     <class name="MSerDeInfo" identity-type="datastore" table="SERDES" detachable="true">
+      <datastore-identity>
+        <column name="SERDE_ID"/>
+      </datastore-identity>
       <field name="name">
         <column name="NAME" length="128" jdbc-type="VARCHAR"/>
       </field>
-      <field name="serializationFormat">
-        <column name="SERIALIZATION_FORMAT" length="1024" jdbc-type="VARCHAR" allows-null="true"/>
-      </field>
-      <field name="serializationClass">
-        <column name="SERIALIZATION_CLASS" length="1024" jdbc-type="VARCHAR" allows-null="true"/>
-      </field>
       <field name="serializationLib">
-        <column name="SERIALIZATION_LIB" length="1024" jdbc-type="VARCHAR" allows-null="true"/>
-      </field>
-      <field name="fieldDelim">
-        <column name="FIELD_DELIM" length="128" jdbc-type="VARCHAR" allows-null="true"/>
-      </field>
-      <field name="collectionItemDelim">
-        <column name="COLLECTION_ITEM_DELIM" length="128" jdbc-type="VARCHAR" allows-null="true"/>
+        <column name="SLIB" length="767" jdbc-type="VARCHAR" allows-null="true"/>
       </field>
-      <field name="mapKeyDelim">
-        <column name="MAP_KEY_DELIM" length="128" jdbc-type="VARCHAR" allows-null="true"/>
-      </field>
-      <field name="lineDelim">
-        <column name="LINE_DELIM" length="128" jdbc-type="VARCHAR" allows-null="true"/>
+      <field name="parameters" table="SERDE_PARAMS">
+        <map key-type="java.lang.String" value-type="java.lang.String"/>
+        <join>
+          <column name="SERDE_ID"/>
+        </join>
+        <key>
+           <column name="PARAM_KEY" length="256" jdbc-type="VARCHAR"/>
+        </key>
+        <value>
+           <column name="PARAM_VALUE" length="767" jdbc-type="VARCHAR"/>
+        </value>
       </field>
     </class>
 
     <class name="MOrder" embedded-only="true" table="SORT_ORDER" detachable="true">
       <field name="col">
-        <column name="COLUMN_NAME" length="128" jdbc-type="VARCHAR"/>
+        <column name="COL_NAME" length="128" jdbc-type="VARCHAR"/>
       </field>
       <field name="order">
         <column name="ORDER" jdbc-type="INTEGER"  allows-null="false"/>
       </field>
     </class>
 
-    <class name="MStorageDescriptor" identity-type="datastore" table="STORAGE_DESC" detachable="true">
+    <class name="MStorageDescriptor" identity-type="datastore" table="SDS" detachable="true">
+      <datastore-identity>
+        <column name="SD_ID"/>
+      </datastore-identity>
       <field name="cols" table="COLUMNS" >
         <collection element-type="MFieldSchema"/>
         <join>
@@ -161,6 +171,7 @@
             <column name="COLUMN_NAME"/>
             <column name="TYPE_NAME"/>
           </primary-key>
+          <column name="SD_ID"/>
         </join>
         <element>
           <embedded>
@@ -174,7 +185,7 @@
         </element>
       </field>
       <field name="location">
-        <column name="LOCATION" length="1024" jdbc-type="VARCHAR"/>
+        <column name="LOCATION" length="767" jdbc-type="VARCHAR"/>
       </field>
       <field name="inputFormat">
         <column name="INPUT_FORMAT" length="256" jdbc-type="VARCHAR"/>
@@ -186,15 +197,21 @@
       <field name="numBuckets">
         <column name="NUM_BUCKETS" jdbc-type="integer"/>
       </field>
-      <field name="serDeInfo" dependent="true"/>
+      <field name="serDeInfo" dependent="true">
+        <column name="SERDE_ID"/>
+      </field>
       <field name="bucketCols" table="BUCKETING_COLS">
         <collection element-type="java.lang.String"/>
-        <join/>
+        <join>
+          <column name="SD_ID"/>
+        </join>
         <element column="BUCKET_COL_NAME"/>
       </field>
       <field name="sortCols" table="SORT_COLS">
         <collection element-type="MOrder"/>
-        <join/>
+        <join>
+          <column name="SD_ID"/>
+        </join>
         <element>
           <embedded>
             <field name="col">
@@ -208,21 +225,32 @@
       </field>
       <field name="parameters" table="SD_PARAMS">
         <map key-type="java.lang.String" value-type="java.lang.String"/>
-        <join/>
+        <join>
+          <column name="SD_ID"/>
+        </join>
         <key>
            <column name="PARAM_KEY" length="256" jdbc-type="VARCHAR"/>
         </key>
         <value>
-           <column name="PARAM_VALUE" length="1024" jdbc-type="VARCHAR"/>
+           <column name="PARAM_VALUE" length="767" jdbc-type="VARCHAR"/>
         </value>
       </field>
     </class>
 
     <class name="MPartition" table="PARTITIONS" identity-type="datastore" detachable="true">
+      <index name="UniquePartition" unique="true">
+        <column name="PART_NAME"/>
+        <column name="TBL_ID"/>
+      </index>
+      <datastore-identity>
+        <column name="PART_ID"/>
+      </datastore-identity>
       <field name="partitionName">
-        <column name="PART_NAME" length="1024" jdbc-type="VARCHAR"/>
+        <column name="PART_NAME" length="767" jdbc-type="VARCHAR"/>
+      </field>
+      <field name="table">
+        <column name="TBL_ID"/>
       </field>
-      <field name="table"/>
       <field name="createTime">
         <column name="CREATE_TIME" jdbc-type="integer"/>
       </field>
@@ -231,19 +259,24 @@
       </field>
       <field name="values" table="PARTITION_KEY_VALS">
         <collection element-type="java.lang.String"/>
-        <join/>
-        <element column="PARTITION_KEY_VAL"/>
+        <join>
+          <column name="PART_ID"/>
+        </join>
+        <element column="PART_KEY_VAL"/>
       </field>
       <field name="sd" dependent="true">
+        <column name="SD_ID"/>
       </field>
       <field name="parameters" table="PARTITION_PARAMS">
         <map key-type="java.lang.String" value-type="java.lang.String"/>
-        <join/>
+        <join>
+          <column name="PART_ID"/>
+        </join>
         <key>
            <column name="PARAM_KEY" length="256" jdbc-type="VARCHAR"/>
         </key>
         <value>
-           <column name="PARAM_VALUE" length="1024" jdbc-type="VARCHAR"/>
+           <column name="PARAM_VALUE" length="767" jdbc-type="VARCHAR"/>
         </value>
       </field>
     </class>

Modified: hadoop/core/trunk/src/contrib/hive/metastore/src/test/org/apache/hadoop/hive/metastore/MetaStoreTestBase.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/metastore/src/test/org/apache/hadoop/hive/metastore/MetaStoreTestBase.java?rev=697291&r1=697290&r2=697291&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/metastore/src/test/org/apache/hadoop/hive/metastore/MetaStoreTestBase.java (original)
+++ hadoop/core/trunk/src/contrib/hive/metastore/src/test/org/apache/hadoop/hive/metastore/MetaStoreTestBase.java Fri Sep 19 16:56:30 2008
@@ -101,7 +101,7 @@
     Properties schema = new Properties();
     schema.setProperty(org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_NAME,name);
     schema.setProperty(org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_DB, db);
-    schema.setProperty(org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_SERDE, org.apache.hadoop.hive.metastore.api.Constants.META_SERDE);
+    //schema.setProperty(org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_SERDE, org.apache.hadoop.hive.metastore.api.Constants.META_SERDE);
     schema.setProperty(org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_COLUMNS, "foo,bar");
     schema.setProperty(org.apache.hadoop.hive.serde.Constants.SERIALIZATION_FORMAT,"\t");
     //Path tPath = whRoot_.suffix("/" + db + ".db/" + name);

Modified: hadoop/core/trunk/src/contrib/hive/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java?rev=697291&r1=697290&r2=697291&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java (original)
+++ hadoop/core/trunk/src/contrib/hive/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java Fri Sep 19 16:56:30 2008
@@ -25,7 +25,7 @@
 import junit.framework.TestCase;
 
 import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.api.Constants;
+import org.apache.hadoop.hive.serde.Constants;
 import org.apache.hadoop.hive.metastore.api.Database;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
 import org.apache.hadoop.hive.metastore.api.Order;
@@ -91,12 +91,12 @@
     assertTrue("Unable to create type " + typeName, ret);
   
     Table tbl = new Table();
-    tbl.setDatabase(dbName);
+    tbl.setDbName(dbName);
     tbl.setTableName(tblName);
     StorageDescriptor sd = new StorageDescriptor(); 
     tbl.setSd(sd);
     sd.setCols(typ1.getFields());
-    sd.setIsCompressed(false);
+    sd.setCompressed(false);
     sd.setNumBuckets(1);
     sd.setParameters(new HashMap<String, String>());
     sd.getParameters().put("test_param_1", "Use this for comments etc");
@@ -104,7 +104,8 @@
     sd.getBucketCols().add("name");
     sd.setSerdeInfo(new SerDeInfo());
     sd.getSerdeInfo().setName(tbl.getTableName());
-    sd.getSerdeInfo().setSerializationFormat("3");
+    sd.getSerdeInfo().setParameters(new HashMap<String, String>());
+    sd.getSerdeInfo().getParameters().put(Constants.SERIALIZATION_FORMAT, "1");
     sd.setSortCols(new ArrayList<Order>());
   
     tbl.setPartitionKeys(new ArrayList<FieldSchema>(2));
@@ -114,7 +115,7 @@
     client.createTable(tbl);
   
     Partition part = new Partition();
-    part.setDatabase(dbName);
+    part.setDbName(dbName);
     part.setTableName(tblName);
     part.setValues(vals);
     part.setParameters(new HashMap<String, String>());
@@ -160,7 +161,7 @@
     Database db = client.getDatabase("test1");
     
     assertEquals("name of returned db is different from that of inserted db", "test1", db.getName());
-    assertEquals("location of the returned db is different from that of inserted db", "strange_loc", db.getLocationUri());
+    assertEquals("location of the returned db is different from that of inserted db", "strange_loc", db.getDescription());
     
     boolean ret2 = client.createDatabase("test2", "another_strange_loc");
     assertTrue("Unable to create the databse", ret2);
@@ -168,7 +169,7 @@
     Database db2 = client.getDatabase("test2");
     
     assertEquals("name of returned db is different from that of inserted db", "test2", db2.getName());
-    assertEquals("location of the returned db is different from that of inserted db", "another_strange_loc", db2.getLocationUri());
+    assertEquals("location of the returned db is different from that of inserted db", "another_strange_loc", db2.getDescription());
     
     List<String> dbs = client.getDatabases();
     
@@ -288,12 +289,12 @@
     assertTrue("Unable to create type " + typeName, ret);
     
     Table tbl = new Table();
-    tbl.setDatabase(dbName);
+    tbl.setDbName(dbName);
     tbl.setTableName(tblName);
     StorageDescriptor sd = new StorageDescriptor();
     tbl.setSd(sd);
     sd.setCols(typ1.getFields());
-    sd.setIsCompressed(false);
+    sd.setCompressed(false);
     sd.setNumBuckets(1);
     sd.setParameters(new HashMap<String, String>());
     sd.getParameters().put("test_param_1", "Use this for comments etc");
@@ -301,30 +302,32 @@
     sd.getBucketCols().add("name");
     sd.setSerdeInfo(new SerDeInfo());
     sd.getSerdeInfo().setName(tbl.getTableName());
-    sd.getSerdeInfo().setSerializationFormat("1");
+    sd.getSerdeInfo().setParameters(new HashMap<String, String>());
+    sd.getSerdeInfo().getParameters().put(org.apache.hadoop.hive.serde.Constants.SERIALIZATION_FORMAT, "1");
     
     client.createTable(tbl);
     
     Table tbl2 = client.getTable(dbName, tblName);
     assertNotNull(tbl2);
-    assertEquals(tbl2.getDatabase(), dbName);
+    assertEquals(tbl2.getDbName(), dbName);
     assertEquals(tbl2.getTableName(), tblName);
     assertEquals(tbl2.getSd().getCols().size(), typ1.getFields().size());
-    assertEquals(tbl2.getSd().isIsCompressed(), false);
+    assertEquals(tbl2.getSd().isCompressed(), false);
     assertEquals(tbl2.getSd().getNumBuckets(), 1);
     assertEquals(tbl2.getSd().getLocation(), tbl.getSd().getLocation());
     assertNotNull(tbl2.getSd().getSerdeInfo());
-    assertEquals(tbl2.getSd().getSerdeInfo().getSerializationFormat(), "1");
+    sd.getSerdeInfo().setParameters(new HashMap<String, String>());
+    sd.getSerdeInfo().getParameters().put(org.apache.hadoop.hive.serde.Constants.SERIALIZATION_FORMAT, "1");
     
     tbl.setTableName(tblName2);
     client.createTable(tbl);
   
     Table tbl3 = client.getTable(dbName, tblName2);
     assertNotNull(tbl3);
-    assertEquals(tbl3.getDatabase(), dbName);
+    assertEquals(tbl3.getDbName(), dbName);
     assertEquals(tbl3.getTableName(), tblName2);
     assertEquals(tbl3.getSd().getCols().size(), typ1.getFields().size());
-    assertEquals(tbl3.getSd().isIsCompressed(), false);
+    assertEquals(tbl3.getSd().isCompressed(), false);
     assertEquals(tbl3.getSd().getNumBuckets(), 1);
     assertEquals(tbl3.getSd().getLocation(), tbl.getSd().getLocation());
     
@@ -369,12 +372,12 @@
       assertTrue("Unable to create type " + typeName, ret);
   
       Table tbl = new Table();
-      tbl.setDatabase(dbName);
+      tbl.setDbName(dbName);
       tbl.setTableName(tblName);
       StorageDescriptor sd = new StorageDescriptor();
       tbl.setSd(sd);
       sd.setCols(typ1.getFields());
-      sd.setIsCompressed(false);
+      sd.setCompressed(false);
       sd.setNumBuckets(1);
       sd.setParameters(new HashMap<String, String>());
       sd.getParameters().put("test_param_1", "Use this for comments etc");
@@ -382,19 +385,20 @@
       sd.getBucketCols().add("name");
       sd.setSerdeInfo(new SerDeInfo());
       sd.getSerdeInfo().setName(tbl.getTableName());
-      sd.getSerdeInfo().setSerializationFormat("9");
+      sd.getSerdeInfo().setParameters(new HashMap<String, String>());
+      sd.getSerdeInfo().getParameters().put(org.apache.hadoop.hive.serde.Constants.SERIALIZATION_FORMAT, "9");
   
       tbl.setPartitionKeys(new ArrayList<FieldSchema>(2));
-      tbl.getPartitionKeys().add(new FieldSchema("ds", Constants.DATE_TYPE_NAME, ""));
-      tbl.getPartitionKeys().add(new FieldSchema("hr", Constants.INT_TYPE_NAME, ""));
+      tbl.getPartitionKeys().add(new FieldSchema("ds", org.apache.hadoop.hive.serde.Constants.DATE_TYPE_NAME, ""));
+      tbl.getPartitionKeys().add(new FieldSchema("hr", org.apache.hadoop.hive.serde.Constants.INT_TYPE_NAME, ""));
   
       client.createTable(tbl);
   
       Table tbl2 = client.getTable(dbName, tblName);
-      assertEquals(tbl2.getDatabase(), dbName);
+      assertEquals(tbl2.getDbName(), dbName);
       assertEquals(tbl2.getTableName(), tblName);
       assertEquals(tbl2.getSd().getCols().size(), typ1.getFields().size());
-      assertFalse(tbl2.getSd().isIsCompressed());
+      assertFalse(tbl2.getSd().isCompressed());
       assertEquals(tbl2.getSd().getNumBuckets(), 1);
   
       assertEquals("Use this for comments etc", tbl2.getSd().getParameters().get("test_param_1"));

Modified: hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/Context.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/Context.java?rev=697291&r1=697290&r2=697291&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/Context.java (original)
+++ hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/Context.java Fri Sep 19 16:56:30 2008
@@ -18,24 +18,20 @@
 
 package org.apache.hadoop.hive.ql;
 
-import java.io.File;
+import java.io.DataInput;
 import java.io.IOException;
 import java.io.FileNotFoundException;
-import java.io.InputStream;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
 
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.exec.Utilities;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.util.StringUtils;
 
 public class Context {
-  private File resFile;
+  private Path resFile;
   private Path resDir;
   private FileSystem fs;
   static final private Log LOG = LogFactory.getLog("hive.ql.Context");
@@ -57,14 +53,14 @@
   /**
    * @return the resFile
    */
-  public File getResFile() {
+  public Path getResFile() {
     return resFile;
   }
 
   /**
    * @param resFile the resFile to set
    */
-  public void setResFile(File resFile) {
+  public void setResFile(Path resFile) {
     this.resFile = resFile;
     resDir = null;
     resDirPaths = null;
@@ -105,10 +101,7 @@
     {
       try
       {
-        FileOutputStream outStream = new FileOutputStream(resFile);
-        outStream.close();
-      } catch (FileNotFoundException e) {
-        LOG.info("Context clear error: " + StringUtils.stringifyException(e));
+      	fs.delete(resFile, false);
       } catch (IOException e) {
         LOG.info("Context clear error: " + StringUtils.stringifyException(e));
       }
@@ -120,7 +113,7 @@
     resDirPaths = null;
   }
 
-  public InputStream getStream() {
+  public DataInput getStream() {
     try
     {
       if (!initialized) {
@@ -128,7 +121,7 @@
         if ((resFile == null) && (resDir == null)) return null;
       
         if (resFile != null)
-          return (InputStream)(new FileInputStream(resFile));
+          return (DataInput)fs.open(resFile);
         
         FileStatus status = fs.getFileStatus(resDir);
         assert status.isDir();
@@ -140,7 +133,7 @@
             resDirPaths[pos++] = resFS.getPath();
         if (pos == 0) return null;
         
-        return (InputStream)fs.open(resDirPaths[resDirFilesNum++]);
+        return (DataInput)fs.open(resDirPaths[resDirFilesNum++]);
       }
       else {
         return getNextStream();
@@ -154,12 +147,12 @@
     }
   }
 
-  private InputStream getNextStream() {
+  private DataInput getNextStream() {
     try
     {
       if (resDir != null && resDirFilesNum < resDirPaths.length && 
           (resDirPaths[resDirFilesNum] != null))
-        return (InputStream)fs.open(resDirPaths[resDirFilesNum++]);
+        return (DataInput)fs.open(resDirPaths[resDirFilesNum++]);
     } catch (FileNotFoundException e) {
       LOG.info("getNextStream error: " + StringUtils.stringifyException(e));
       return null;

Modified: hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/Driver.java?rev=697291&r1=697290&r2=697291&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/Driver.java (original)
+++ hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/Driver.java Fri Sep 19 16:56:30 2008
@@ -18,7 +18,8 @@
 
 package org.apache.hadoop.hive.ql;
 
-import java.io.InputStream;
+import java.io.DataInput;
+import java.io.IOException;
 import java.io.Serializable;
 import java.util.*;
 import org.antlr.runtime.tree.CommonTree;
@@ -37,6 +38,7 @@
 import org.apache.hadoop.hive.ql.exec.TaskFactory;
 import org.apache.hadoop.hive.ql.exec.Utilities;
 import org.apache.hadoop.hive.ql.exec.ExecDriver;
+import org.apache.hadoop.hive.serde.ByteStream;
 import org.apache.hadoop.hive.conf.HiveConf;
 
 import org.apache.commons.logging.Log;
@@ -45,23 +47,14 @@
 public class Driver implements CommandProcessor {
 
   static final private Log LOG = LogFactory.getLog("hive.ql.Driver");
-  static final private int separator  = Utilities.ctrlaCode;
-  static final private int terminator = Utilities.newLineCode;
   static final private int MAX_ROWS   = 100;
+  ByteStream.Output bos = new ByteStream.Output();
   
-  private ParseDriver pd;
-  private HiveConf conf;
-  private InputStream resStream;
-  private LogHelper console;
-  private Context   ctx;
-  
-  public static int getSeparator() {
-    return separator;
-  }
-
-  public static int getTerminator() {
-    return terminator;
-  }
+  private ParseDriver  pd;
+  private HiveConf     conf;
+  private DataInput    resStream;
+  private LogHelper    console;
+  private Context      ctx;
   
   public int countJobs(List<Task<? extends Serializable>> tasks) {
     if (tasks == null)
@@ -108,12 +101,9 @@
       BaseSemanticAnalyzer sem;
       LOG.info("Starting command: " + command);
 
-      if (resStream != null) {
-        resStream.close();
-        resStream = null;
-      }
       ctx.clear();
-
+      resStream = null;
+      
       pd = new ParseDriver();
       CommonTree tree = pd.parse(command);
 
@@ -200,95 +190,71 @@
         conf.setVar(HiveConf.ConfVars.HADOOPJOBNAME, "");
       } 
     }
-    if (jobs > 0) {
-      console.printInfo("OK");
-    }
+
+    console.printInfo("OK");
     return (0);
   }
   
   
-  public boolean getResults(Vector<Vector<String>> res) {
-
+  public boolean getResults(Vector<Vector<String>> res) 
+  {
     if (resStream == null)
       resStream = ctx.getStream();
     if (resStream == null) return false;
     
-    int sizeArr = 128;
-    char[] tmpCharArr = new char[sizeArr];
-    
-    for (int numRows = 0; numRows < MAX_ROWS; numRows++)
+    int numRows = 0;
+    Vector<String> row = new Vector<String>();
+
+    while (numRows < MAX_ROWS)
     {
-      if (resStream == null) {
-        if (numRows > 0) {
+      if (resStream == null) 
+      {
+        if (numRows > 0)
           return true;
-        }
-        else {
+        else
           return false;
-        }
       }
-      boolean eof = false;
-      Vector<String> row = new Vector<String>();
-      String col;
-      int len = 0;
-      while (true) {
-        char c;
-        try {
-          int i = resStream.read();
-          if (i == -1)
-          {
-            eof = true;
-            break;
-          }
-          
-          c = (char)i;
-          
-          if (c == terminator) {
-            col = new String(tmpCharArr, 0, len);
-            len = 0;
-            row.add(col.equals(Utilities.nullStringStorage) ? null : col);
-            res.add(row);
-            break;
-          }
-          else if (c == separator) {
-            col = new String(tmpCharArr, 0, len);
-            len = 0;
-            row.add(col.equals(Utilities.nullStringStorage) ? null : col);
-          }
-          else
-          {
-            if (sizeArr == len)
-            {
-              char[] tmp = new char[2*sizeArr];
-              sizeArr *= 2;
-              for (int idx = 0; idx < len; idx++)
-                tmp[idx] = tmpCharArr[idx];
-              tmpCharArr = tmp;
-            }
-            tmpCharArr[len++] = c;
-          }
-          
-        } 
-        catch (java.io.IOException e) {
-          console.printError("FAILED: Unknown exception : " + e.getMessage(),
-                             "\n" + org.apache.hadoop.util.StringUtils.stringifyException(e));
-          return false;
-        }
+
+      String col = null;
+      bos.reset();
+      Utilities.streamStatus ss = Utilities.streamStatus.NORMAL;
+      try
+      {
+        ss = Utilities.readColumn(resStream, bos);
+        if (bos.getCount() > 0)
+          col = new String(bos.getData(), 0, bos.getCount(), "UTF-8");
+        else if (ss == Utilities.streamStatus.NORMAL)
+          col = Utilities.NSTR;
+      } catch (IOException e) {
+        console.printError("FAILED: Unexpected IO exception : " + e.getMessage());
+        res = null;
+        return false;
       }
       
-      if (eof)
+      if ((ss == Utilities.streamStatus.EOF) || 
+          (ss == Utilities.streamStatus.TERMINATED))
       {
-        if (len > 0)
-        {
-          col = new String(tmpCharArr, 0, len);
-          len = 0;
+        if (col != null) 
           row.add(col.equals(Utilities.nullStringStorage) ? null : col);
-          res.add(row);
-        }
+        else if (row.size() != 0) 
+          row.add(null);
+
+        numRows++;
+        res.add(row);
+        row = new Vector<String>();
+        col = null;
 
-        resStream = ctx.getStream();
+        if (ss == Utilities.streamStatus.EOF) 
+          resStream = ctx.getStream();
       }
+      else if (ss == Utilities.streamStatus.NORMAL)
+      {
+        row.add(col.equals(Utilities.nullStringStorage) ? null : col);
+        col = null;
+      }
+      else
+        assert false;
     }
-    
     return true;
   }
 }

Modified: hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/CollectOperator.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/CollectOperator.java?rev=697291&r1=697290&r2=697291&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/CollectOperator.java (original)
+++ hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/CollectOperator.java Fri Sep 19 16:56:30 2008
@@ -23,6 +23,9 @@
 
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.ql.plan.collectDesc;
+import org.apache.hadoop.hive.serde2.objectinspector.InspectableObject;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
 import org.apache.hadoop.conf.Configuration;
 
 /**
@@ -31,26 +34,41 @@
 public class CollectOperator extends Operator <collectDesc> implements Serializable {
 
   private static final long serialVersionUID = 1L;
-  transient protected ArrayList<HiveObject> objList;
+  transient protected ArrayList<Object> rowList;
+  transient protected ArrayList<ObjectInspector> rowInspectorList;
   transient int maxSize;
 
   public void initialize(Configuration hconf) throws HiveException {
     super.initialize(hconf);
-    objList = new ArrayList<HiveObject> ();
+    rowList = new ArrayList<Object> ();
+    rowInspectorList = new ArrayList<ObjectInspector> ();
     maxSize = conf.getBufferSize().intValue();
   }
 
-  public void process(HiveObject r) throws HiveException {
-    if(objList.size() < maxSize) {
-      objList.add(r);
+  public void process(Object row, ObjectInspector rowInspector)
+      throws HiveException {
+    if(rowList.size() < maxSize) {
+      // Create a standard copy of the object.
+      // In the future we can optimize this by doing copy-on-write.
+      // Here we always copy the object so that other operators can reuse the object for the next row. 
+      Object o = ObjectInspectorUtils.getStandardObject(row, rowInspector);
+      ObjectInspector oi = ObjectInspectorUtils.getStandardObjectInspector(rowInspector);
+      rowList.add(o);
+      rowInspectorList.add(oi);
     }
-    forward(r);
+    forward(row, rowInspector);
   }
   
-  public HiveObject retrieve() {
-    if(objList.isEmpty())
-      return null;
-    return objList.remove(0);
+  public void retrieve(InspectableObject result) {
+    assert(result != null);
+    if (rowList.isEmpty()) {
+      result.o = null;
+      result.oi = null;
+    } else {
+      result.o = rowList.remove(0);
+      result.oi = rowInspectorList.remove(0);
+    }
   }
 
+
 }

Modified: hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnInfo.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnInfo.java?rev=697291&r1=697290&r2=697291&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnInfo.java (original)
+++ hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnInfo.java Fri Sep 19 16:56:30 2008
@@ -21,7 +21,8 @@
 import java.lang.Class;
 import java.io.*;
 
-import org.apache.hadoop.hive.ql.parse.TypeInfo;
+import org.apache.hadoop.hive.ql.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.ql.typeinfo.TypeInfoFactory;
 
 /**
  * Implementation for ColumnInfo which contains the internal name for the 
@@ -55,7 +56,7 @@
   
   public ColumnInfo(String internalName, Class type, boolean isVirtual) {
     this.internalName = internalName;
-    this.type = TypeInfo.getPrimitiveTypeInfo(type);
+    this.type = TypeInfoFactory.getPrimitiveTypeInfo(type);
     this.isVirtual = isVirtual;
   }
   

Modified: hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/CompositeHiveObject.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/CompositeHiveObject.java?rev=697291&r1=697290&r2=697291&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/CompositeHiveObject.java (original)
+++ hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/CompositeHiveObject.java Fri Sep 19 16:56:30 2008
@@ -1,178 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.exec;
-
-import java.util.*;
-import org.apache.hadoop.hive.serde.*;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
-
-/**
- * fixed width composition of Hive Objects
- */
-public class CompositeHiveObject extends HiveObject {
-
-  public static class CompositeSerDeField implements SerDeField {
-    int position;
-    String expr;
-    protected SerDeField field;
-
-    public CompositeSerDeField (int position, String expr) {
-      this.position = position;
-      this.expr = expr;
-    }
-
-    public HiveObject getHiveObject(Object obj) throws SerDeException {
-      ArrayList<HiveObject>  hol = (ArrayList<HiveObject>)obj;
-      HiveObject ho = hol.get(this.position);
-
-      if(ho == null)
-        return new NullHiveObject();
-      
-      if(this.expr == null) {
-        // no need to descend any further
-        return ho;
-      }
-
-      try {
-        if (this.field == null) {
-          this.field = ho.getFieldFromExpression(this.expr);
-        }
-        // descend recursively
-        return (ho.get(this.field));
-      } catch (HiveException e) {
-        throw new SerDeException (e);
-      }
-    }
-
-    public Object get(Object obj) throws SerDeException {
-      throw new RuntimeException ("get() cannot be called for CompositeSerDeField");
-    }
-    public boolean isList() { throw new RuntimeException("Not supported"); }
-    public boolean isMap() { throw new RuntimeException("Not supported"); }
-    public boolean isPrimitive() { throw new RuntimeException("Not supported"); }
-    public Class getType() { throw new RuntimeException("Not supported"); }
-    public Class getListElementType() {throw new RuntimeException("Not supported"); }
-    public Class getMapKeyType() { throw new RuntimeException("Not supported"); }
-    public Class getMapValueType() { throw new RuntimeException("Not supported"); }
-    public String getName() {  throw new RuntimeException("Not supported"); }
-  }
-
-  ArrayList<HiveObject> hol;
-  SerDe csd;
-  int width;
-
-  public CompositeHiveObject(int width) {
-    assert(width > 0);
-    this.hol = new ArrayList<HiveObject> (width);
-    this.width = width;
-  }
-
-  public CompositeHiveObject(ArrayList<HiveObject> hol) {
-    this.width = hol.size();
-    this.hol = hol;
-  }
-  
-  @Override
-  public SerDeField getFieldFromExpression(String compositeExpr) throws HiveException {
-    int position = this.width+1;
-    String expr = null;
-    int dot = compositeExpr.indexOf(".");
-    try {
-      if(dot == -1) {
-        position = Integer.parseInt(compositeExpr);
-        expr = null;
-      } else {
-        position = Integer.parseInt(compositeExpr.substring(0, dot));
-        expr = compositeExpr.substring(dot+1);
-        if(expr.equals("")) { expr = null; }
-      }
-    } catch (NumberFormatException e) {
-      throw new HiveException("Field Name is not a number: "+compositeExpr);
-    }
-    
-    if((position >= this.width) || (position < 0)) {
-      throw new HiveException("Index=" + position + " is not between 0 and (width=)" + this.width);
-    }
-    
-    return new CompositeSerDeField(position, expr);
-  }
-
-  @Override
-  public HiveObject get(SerDeField field) throws HiveException {
-    try {
-      CompositeSerDeField csdField = (CompositeSerDeField)field;
-      return (csdField.getHiveObject(this.hol));
-    } catch (SerDeException e) {
-      throw new HiveException(e);
-    }
-  }
-
-  @Override
-  public List<SerDeField> getFields() throws HiveException {
-    ArrayList<SerDeField> ret = new ArrayList<SerDeField> (this.width);
-    for(int i=0; i<this.width; i++) {
-      ret.add(new CompositeSerDeField(i, null));
-    }
-    return (ret);
-  }
-
-  public static List<SerDeField> getFields(int width) throws HiveException {
-    ArrayList<SerDeField> ret = new ArrayList<SerDeField> (width);
-    for(int i=0; i<width; i++) {
-      ret.add(new CompositeSerDeField(i, null));
-    }
-    return (ret);
-  }
-
-  @SuppressWarnings("nls")
-  public void addHiveObject(HiveObject ho) throws HiveException {
-    if(this.hol.size() >= this.width) {
-      throw new HiveException("Exceeded max size of Composite Hive Object ="+this.width);
-    }
-    this.hol.add(ho);
-  }
-
-  @Override
-  public Object getJavaObject() throws HiveException {
-    ArrayList result = new ArrayList();
-    for(HiveObject ho: hol) {
-      result.add(ho == null ? null : ho.getJavaObject());
-    }
-    return result;
-  }
-
-  @Override
-  public boolean isPrimitive() { return false; }
-
-  @Override
-  public boolean equals(Object other) {
-    if (!(other instanceof CompositeHiveObject)) return false;
-    CompositeHiveObject choOther = (CompositeHiveObject) other;
-    return this.hol.equals(choOther.hol);
-  }
-
-  @Override
-  public int hashCode() {
-    int ret = 12345;
-    for(HiveObject ho: this.hol) {
-      ret = ret * 31 + ho.hashCode();
-    }
-    return ret;
-  }
-}

Modified: hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java?rev=697291&r1=697290&r2=697291&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java (original)
+++ hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java Fri Sep 19 16:56:30 2008
@@ -18,8 +18,8 @@
 
 package org.apache.hadoop.hive.ql.exec;
 
+import java.io.DataOutput;
 import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.Serializable;
 import java.util.Iterator;
@@ -27,7 +27,10 @@
 import java.util.SortedSet;
 import java.util.TreeSet;
 
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.serde.Constants;
 import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
 import org.apache.hadoop.hive.metastore.api.MetaException;
 import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
@@ -47,6 +50,9 @@
 import org.apache.hadoop.mapred.TextInputFormat;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.hive.ql.metadata.InvalidTableException;
+import org.apache.hadoop.hive.metastore.api.Order;
+import org.apache.hadoop.hive.serde.Constants;
+import org.apache.hadoop.util.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import com.facebook.thrift.TException;
@@ -72,8 +78,10 @@
 
     // Create the db
     Hive db;
+    FileSystem fs;
     try {
       db = Hive.get(conf);
+      fs = FileSystem.get(conf);
 
       createTableDesc crtTbl = work.getCreateTblDesc();
       if (crtTbl != null) {
@@ -91,21 +99,19 @@
         if (crtTbl.getNumBuckets() != -1)
           tblStorDesc.setNumBuckets(crtTbl.getNumBuckets());
         if (crtTbl.getFieldDelim() != null)
-          tbl.setFieldDelim(crtTbl.getFieldDelim());
-
+          tbl.setSerdeParam(Constants.FIELD_DELIM, crtTbl.getFieldDelim());
         if (crtTbl.getCollItemDelim() != null)
-          tbl.setCollectionItemDelim(crtTbl.getCollItemDelim());
+          tbl.setSerdeParam(Constants.COLLECTION_DELIM, crtTbl.getCollItemDelim());
         if (crtTbl.getMapKeyDelim() != null)
-          tbl.setMapKeyDelim(crtTbl.getMapKeyDelim());
+          tbl.setSerdeParam(Constants.MAPKEY_DELIM, crtTbl.getMapKeyDelim());
         if (crtTbl.getLineDelim() != null)
-          tbl.setLineDelim(crtTbl.getLineDelim());
+          tbl.setSerdeParam(Constants.LINE_DELIM, crtTbl.getLineDelim());
         if (crtTbl.getComment() != null)
           tbl.setProperty("comment", crtTbl.getComment());
         if (crtTbl.getLocation() != null)
           tblStorDesc.setLocation(crtTbl.getLocation());
 
-        tbl.setIsCompressed(crtTbl.isCompressed());
-        if (crtTbl.isCompressed()) {
+        if (crtTbl.isSequenceFile()) {
           tbl.setInputFormatClass(SequenceFileInputFormat.class);
           tbl.setOutputFormatClass(SequenceFileOutputFormat.class);
         }
@@ -116,7 +122,42 @@
 
         if (crtTbl.isExternal())
           tbl.setProperty("EXTERNAL", "TRUE");
-        
+
+        // If the sorted columns is a superset of bucketed columns, store this fact. It can be later used to
+        // optimize some group-by queries. Note that, the order does not matter as long as it in the first
+        // 'n' columns where 'n' is the length of the bucketed columns.
+        if ((tbl.getBucketCols() != null) && (tbl.getSortCols() != null))
+        {
+          List<String> bucketCols = tbl.getBucketCols();
+          List<Order> sortCols = tbl.getSortCols();
+
+          if (sortCols.size() >= bucketCols.size())
+          {
+            boolean found = true;
+
+            Iterator<String> iterBucketCols = bucketCols.iterator();
+            while (iterBucketCols.hasNext())
+            {
+              String bucketCol = iterBucketCols.next();
+              boolean colFound = false;
+              for (int i = 0; i < bucketCols.size(); i++)
+              {
+                if (bucketCol.equals(sortCols.get(i).getCol())) {
+                  colFound = true;
+                  break;
+                }
+              }
+              if (colFound == false)
+              {
+                found = false;
+                break;
+              }
+            }
+            if (found)
+              tbl.setProperty("SORTBUCKETCOLSPREFIX", "TRUE");
+          }
+        }
+
         // create the table
         db.createTable(tbl);
         return 0;
@@ -162,21 +203,24 @@
           LOG.info("DDLTask: got data for " +  tbl.getName());
           
           // write the results in the file
-          FileOutputStream outStream = new FileOutputStream(descTbl.getResFile());
+          DataOutput os = (DataOutput)fs.create(descTbl.getResFile());
           List<FieldSchema> cols = tbl.getCols();
           Iterator<FieldSchema> iterCols = cols.iterator();
+          boolean firstCol = true;
           while (iterCols.hasNext())
           {
+            if (!firstCol)
+              os.write(terminator);
             FieldSchema col = iterCols.next();
-            outStream.write(col.getName().getBytes());
-            outStream.write(separator);
-            outStream.write(col.getType().getBytes());
+            os.write(col.getName().getBytes("UTF-8"));
+            os.write(separator);
+            os.write(col.getType().getBytes("UTF-8"));
             if (col.getComment() != null)
             {
-              outStream.write(separator);
-              outStream.write(col.getComment().getBytes());
+              os.write(separator);
+              os.write(col.getComment().getBytes("UTF-8"));
             }
-            outStream.write(terminator);
+            firstCol = false;
           }
 
           // also return the partitioning columns
@@ -184,19 +228,20 @@
           Iterator<FieldSchema> iterPartCols = partCols.iterator();
           while (iterPartCols.hasNext())
           {
+            os.write(terminator);
             FieldSchema col = iterPartCols.next();
-            outStream.write(col.getName().getBytes());
-            outStream.write(separator);
-            outStream.write(col.getType().getBytes());
+            os.write(col.getName().getBytes("UTF-8"));
+            os.write(separator);
+            os.write(col.getType().getBytes("UTF-8"));
             if (col.getComment() != null)
             {
-              outStream.write(separator);
-              outStream.write(col.getComment().getBytes());
+              os.write(separator);
+              os.write(col.getComment().getBytes("UTF-8"));
             }
-            outStream.write(terminator);
           }
-          outStream.close();
           LOG.info("DDLTask: written data for " +  tbl.getName());
+          ((FSDataOutputStream)os).close();
+          
         } catch (FileNotFoundException e) {
           LOG.info("describe table: " + StringUtils.stringifyException(e));
           return 1;
@@ -212,10 +257,10 @@
         if (!found)
         {
           try {
-            FileOutputStream outStream = new FileOutputStream(descTbl.getResFile());
+            DataOutput outStream = (DataOutput)fs.open(descTbl.getResFile());
             String errMsg = "Table " + descTbl.getTableName() + " does not exist";
-            outStream.write(errMsg.getBytes());
-            outStream.close();
+            outStream.write(errMsg.getBytes("UTF-8"));
+            ((FSDataOutputStream)outStream).close();
           } catch (FileNotFoundException e) {
             LOG.info("describe table: " + StringUtils.stringifyException(e));
             return 1;
@@ -243,7 +288,7 @@
         
         // write the results in the file
         try {
-          FileOutputStream outStream = new FileOutputStream(showTbls.getResFile());
+          DataOutput outStream = (DataOutput)fs.create(showTbls.getResFile());
           SortedSet<String> sortedTbls = new TreeSet<String>(tbls);
           Iterator<String> iterTbls = sortedTbls.iterator();
           boolean firstCol = true;
@@ -251,10 +296,10 @@
           {
             if (!firstCol)
               outStream.write(separator);
-            outStream.write(iterTbls.next().getBytes());
+            outStream.write(iterTbls.next().getBytes("UTF-8"));
             firstCol = false;
           }
-          outStream.write(terminator);
+          ((FSDataOutputStream)outStream).close();
         } catch (FileNotFoundException e) {
           LOG.info("show table: " + StringUtils.stringifyException(e));
           return 1;

Modified: hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java?rev=697291&r1=697290&r2=697291&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java (original)
+++ hadoop/core/trunk/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java Fri Sep 19 16:56:30 2008
@@ -147,15 +147,8 @@
     FileOutputFormat.setOutputPath(job, new Path(jobScratchDir));
     job.setMapperClass(ExecMapper.class);
 
-    if(!work.getNeedsTagging()) {
-      job.setMapOutputValueClass(NoTagWritableHiveObject.class);
-      job.setMapOutputKeyClass(NoTagWritableComparableHiveObject.class);    
-      job.setOutputKeyComparatorClass(NoTagHiveObjectComparator.class);
-    } else {
-      job.setMapOutputValueClass(WritableHiveObject.class);
-      job.setMapOutputKeyClass(WritableComparableHiveObject.class);    
-      job.setOutputKeyComparatorClass(HiveObjectComparator.class);
-    }
+    job.setMapOutputValueClass(Text.class);
+    job.setMapOutputKeyClass(HiveKey.class);    
 
     job.setNumReduceTasks(work.getNumReduceTasks().intValue());
     job.setReducerClass(ExecReducer.class);



Mime
View raw message