incubator-hcatalog-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ga...@apache.org
Subject svn commit: r1091509 [3/8] - in /incubator/hcatalog/trunk: ./ bin/ ivy/ src/ src/docs/ src/docs/src/ src/docs/src/documentation/ src/docs/src/documentation/classes/ src/docs/src/documentation/conf/ src/docs/src/documentation/content/ src/docs/src/docum...
Date Tue, 12 Apr 2011 17:30:12 GMT
Added: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/AuthUtils.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/AuthUtils.java?rev=1091509&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/AuthUtils.java (added)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/AuthUtils.java Tue Apr 12 17:30:08 2011
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.common;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+import javax.security.auth.login.LoginException;
+
+import org.apache.commons.lang.ArrayUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.shims.ShimLoader;
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.UserGroupInformation;
+
+public class AuthUtils {
+
+  /**
+   * @param path non-null
+   * @param action non-null
+   * @param conf
+   * @throws SemanticException
+   * @throws HCatException
+   *
+   * This method validates only for existing path. If path doesn't exist
+   * there is nothing to validate. So, make sure that path passed in is non-null.
+   */
+
+  @SuppressWarnings("deprecation")
+  public static void authorize(final Path path, final FsAction action, final Configuration conf) throws SemanticException, HCatException{
+
+    if(path == null) {
+      throw new HCatException(ErrorType.ERROR_INTERNAL_EXCEPTION);
+    }
+    final FileStatus stat;
+
+    try {
+      stat = path.getFileSystem(conf).getFileStatus(path);
+    } catch (FileNotFoundException fnfe){
+      // File named by path doesn't exist; nothing to validate.
+      return;
+    }
+    catch (AccessControlException ace) {
+      throw new HCatException(ErrorType.ERROR_ACCESS_CONTROL, ace);
+    } catch (org.apache.hadoop.fs.permission.AccessControlException ace){
+      // Older hadoop version will throw this @deprecated Exception.
+      throw new HCatException(ErrorType.ERROR_ACCESS_CONTROL, ace);
+    } catch (IOException ioe){
+      throw new SemanticException(ioe);
+    }
+
+    final UserGroupInformation ugi;
+    try {
+      ugi = ShimLoader.getHadoopShims().getUGIForConf(conf);
+    } catch (LoginException le) {
+      throw new HCatException(ErrorType.ERROR_ACCESS_CONTROL,le);
+    } catch (IOException ioe) {
+      throw new SemanticException(ioe);
+    }
+
+    final FsPermission dirPerms = stat.getPermission();
+
+    final String user = HiveConf.getBoolVar(conf, ConfVars.METASTORE_USE_THRIFT_SASL) ?
+                          ugi.getShortUserName() : ugi.getUserName();
+    final String grp = stat.getGroup();
+    if(user.equals(stat.getOwner())){
+      if(dirPerms.getUserAction().implies(action)){
+        return;
+      }
+      throw new HCatException(ErrorType.ERROR_ACCESS_CONTROL);
+    }
+    if(ArrayUtils.contains(ugi.getGroupNames(), grp)){
+      if(dirPerms.getGroupAction().implies(action)){
+        return;
+      }
+      throw new HCatException(ErrorType.ERROR_ACCESS_CONTROL);
+
+    }
+    if(dirPerms.getOtherAction().implies(action)){
+      return;
+    }
+    throw new HCatException(ErrorType.ERROR_ACCESS_CONTROL);
+
+
+  }
+}

Added: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/ErrorType.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/ErrorType.java?rev=1091509&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/ErrorType.java (added)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/ErrorType.java Tue Apr 12 17:30:08 2011
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.common;
+
+/**
+ * Enum type representing the various errors throws by Howl.
+ */
+public enum ErrorType {
+
+    /* Howl Input Format related errors 1000 - 1999 */
+    ERROR_DB_INIT                       (1000, "Error initializing database session"),
+    ERROR_EXCEED_MAXPART                (1001, "Query result exceeded maximum number of partitions allowed"),
+
+
+    /* Howl Output Format related errors 2000 - 2999 */
+    ERROR_INVALID_TABLE                 (2000, "Table specified does not exist"),
+    ERROR_SET_OUTPUT                    (2001, "Error setting output information"),
+    ERROR_DUPLICATE_PARTITION           (2002, "Partition already present with given partition key values"),
+    ERROR_NON_EMPTY_TABLE               (2003, "Non-partitioned table already contains data"),
+    ERROR_NOT_INITIALIZED               (2004, "HowlOutputFormat not initialized, setOutput has to be called"),
+    ERROR_INIT_STORAGE_DRIVER           (2005, "Error initializing output storage driver instance"),
+    ERROR_PUBLISHING_PARTITION          (2006, "Error adding partition to metastore"),
+    ERROR_SCHEMA_COLUMN_MISMATCH        (2007, "Invalid column position in partition schema"),
+    ERROR_SCHEMA_PARTITION_KEY          (2008, "Partition key cannot be present in the partition data"),
+    ERROR_SCHEMA_TYPE_MISMATCH          (2009, "Invalid column type in partition schema"),
+    ERROR_INVALID_PARTITION_VALUES      (2010, "Invalid partition values specified"),
+    ERROR_MISSING_PARTITION_KEY         (2011, "Partition key value not provided for publish"),
+    ERROR_MOVE_FAILED                   (2012, "Moving of data failed during commit"),
+
+
+    /* Authorization Errors 3000 - 3999 */
+    ERROR_ACCESS_CONTROL           (3000, "Permission denied"),
+
+    /* Miscellaneous errors, range 9000 - 9998 */
+    ERROR_UNIMPLEMENTED                 (9000, "Functionality currently unimplemented"),
+    ERROR_INTERNAL_EXCEPTION            (9001, "Exception occurred while processing Howl request");
+
+    /** The error code. */
+    private int errorCode;
+
+    /** The error message. */
+    private String errorMessage;
+
+    /** Should the causal exception message be appended to the error message, yes by default*/
+    private boolean appendCauseMessage = true;
+
+    /** Is this a retriable error, no by default. */
+    private boolean isRetriable = false;
+
+    /**
+     * Instantiates a new error type.
+     * @param errorCode the error code
+     * @param errorMessage the error message
+     */
+    private ErrorType(int errorCode, String errorMessage) {
+        this.errorCode = errorCode;
+        this.errorMessage = errorMessage;
+    }
+
+    /**
+     * Instantiates a new error type.
+     * @param errorCode the error code
+     * @param errorMessage the error message
+     * @param appendCauseMessage should causal exception message be appended to error message
+     */
+    private ErrorType(int errorCode, String errorMessage, boolean appendCauseMessage) {
+        this.errorCode = errorCode;
+        this.errorMessage = errorMessage;
+        this.appendCauseMessage = appendCauseMessage;
+    }
+
+    /**
+     * Instantiates a new error type.
+     * @param errorCode the error code
+     * @param errorMessage the error message
+     * @param appendCauseMessage should causal exception message be appended to error message
+     * @param isRetriable is this a retriable error
+     */
+    private ErrorType(int errorCode, String errorMessage, boolean appendCauseMessage, boolean isRetriable) {
+        this.errorCode = errorCode;
+        this.errorMessage = errorMessage;
+        this.appendCauseMessage = appendCauseMessage;
+        this.isRetriable = isRetriable;
+    }
+
+    /**
+     * Gets the error code.
+     * @return the error code
+     */
+    public int getErrorCode() {
+        return errorCode;
+    }
+
+    /**
+     * Gets the error message.
+     * @return the error message
+     */
+    public String getErrorMessage() {
+        return errorMessage;
+    }
+
+    /**
+     * Checks if this is a retriable error.
+     * @return true, if is a retriable error, false otherwise
+     */
+    public boolean isRetriable() {
+        return isRetriable;
+    }
+
+    /**
+     * Whether the cause of the exception should be added to the error message.
+     * @return true, if the cause should be added to the message, false otherwise
+     */
+    public boolean appendCauseMessage() {
+        return appendCauseMessage;
+    }
+}

Added: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatConstants.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatConstants.java?rev=1091509&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatConstants.java (added)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatConstants.java Tue Apr 12 17:30:08 2011
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.common;
+
+public final class HCatConstants {
+
+  /** The key for the input storage driver class name */
+  public static final String HCAT_ISD_CLASS = "hcat.isd";
+
+  /** The key for the output storage driver class name */
+  public static final String HCAT_OSD_CLASS = "hcat.osd";
+
+  public static final String HIVE_RCFILE_IF_CLASS = "org.apache.hadoop.hive.ql.io.RCFileInputFormat";
+  public static final String HIVE_RCFILE_OF_CLASS = "org.apache.hadoop.hive.ql.io.RCFileOutputFormat";
+  public static final String HCAT_RCFILE_ISD_CLASS = "org.apache.hcatalog.rcfile.RCFileInputDriver";
+  public static final String HCAT_RCFILE_OSD_CLASS = "org.apache.hcatalog.rcfile.RCFileOutputDriver";
+
+  //The keys used to store info into the job Configuration
+  public static final String HCAT_KEY_BASE = "mapreduce.lib.hcat";
+
+  public static final String HCAT_KEY_OUTPUT_SCHEMA = HCAT_KEY_BASE + ".output.schema";
+
+  public static final String HCAT_KEY_JOB_INFO =  HCAT_KEY_BASE + ".job.info";
+
+  private HCatConstants() { // restrict instantiation
+  }
+
+  public static final String HCAT_TABLE_SCHEMA = "hcat.table.schema";
+
+  public static final String HCAT_METASTORE_URI = "hcat.metastore.uri";
+
+  public static final String HCAT_PERMS = "hcat.perms";
+
+  public static final String HCAT_GROUP = "hcat.group";
+
+  public static final String HCAT_CREATE_TBL_NAME = "hcat.create.tbl.name";
+
+  public static final String HCAT_CREATE_DB_NAME = "hcat.create.db.name";
+
+  public static final String HCAT_METASTORE_PRINCIPAL = "hcat.metastore.principal";
+
+  // IMPORTANT IMPORTANT IMPORTANT!!!!!
+  //The keys used to store info into the job Configuration.
+  //If any new keys are added, the HowlStorer needs to be updated. The HowlStorer
+  //updates the job configuration in the backend to insert these keys to avoid
+  //having to call setOutput from the backend (which would cause a metastore call
+  //from the map jobs)
+  public static final String HCAT_KEY_OUTPUT_BASE = "mapreduce.lib.hcatoutput";
+  public static final String HCAT_KEY_OUTPUT_INFO = HCAT_KEY_OUTPUT_BASE + ".info";
+  public static final String HCAT_KEY_HIVE_CONF = HCAT_KEY_OUTPUT_BASE + ".hive.conf";
+  public static final String HCAT_KEY_TOKEN_SIGNATURE = HCAT_KEY_OUTPUT_BASE + ".token.sig";
+}

Added: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatException.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatException.java?rev=1091509&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatException.java (added)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatException.java Tue Apr 12 17:30:08 2011
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.common;
+
+import java.io.IOException;
+
+/**
+ * Class representing exceptions thrown by Howl.
+ */
+public class HCatException extends IOException {
+
+  private static final long serialVersionUID = 1L;
+
+  /** The error type enum for this exception. */
+  private final ErrorType errorType;
+
+  /**
+   * Instantiates a new howl exception.
+   * @param errorType the error type
+   */
+  public HCatException(ErrorType errorType) {
+    this(errorType, null, null);
+  }
+
+
+  /**
+   * Instantiates a new howl exception.
+   * @param errorType the error type
+   * @param cause the cause
+   */
+  public HCatException(ErrorType errorType, Throwable cause) {
+    this(errorType, null, cause);
+  }
+
+  /**
+   * Instantiates a new howl exception.
+   * @param errorType the error type
+   * @param extraMessage extra messages to add to the message string
+   */
+  public HCatException(ErrorType errorType, String extraMessage) {
+    this(errorType, extraMessage, null);
+  }
+
+  /**
+   * Instantiates a new howl exception.
+   * @param errorType the error type
+   * @param extraMessage extra messages to add to the message string
+   * @param cause the cause
+   */
+  public HCatException(ErrorType errorType, String extraMessage, Throwable cause) {
+    super(buildErrorMessage(
+        errorType,
+        extraMessage,
+        cause), cause);
+    this.errorType = errorType;
+  }
+
+
+  //TODO : remove default error type constructors after all exceptions
+  //are changed to use error types
+  /**
+   * Instantiates a new howl exception.
+   * @param message the error message
+   */
+  public HCatException(String message) {
+    this(ErrorType.ERROR_INTERNAL_EXCEPTION, message, null);
+  }
+
+  /**
+   * Instantiates a new howl exception.
+   * @param message the error message
+   * @param cause the cause
+   */
+  public HCatException(String message, Throwable cause) {
+    this(ErrorType.ERROR_INTERNAL_EXCEPTION, message, cause);
+  }
+
+
+  /**
+   * Builds the error message string. The error type message is appended with the extra message. If appendCause
+   * is true for the error type, then the message of the cause also is added to the message.
+   * @param type the error type
+   * @param extraMessage the extra message string
+   * @param cause the cause for the exception
+   * @return the exception message string
+   */
+  public static String buildErrorMessage(ErrorType type, String extraMessage, Throwable cause) {
+
+    //Initial message is just the error type message
+    StringBuffer message = new StringBuffer(HCatException.class.getName());
+    message.append(" : " + type.getErrorCode());
+    message.append(" : " + type.getErrorMessage());
+
+    if( extraMessage != null ) {
+      //Add the extra message value to buffer
+      message.append(" : " + extraMessage);
+    }
+
+    if( type.appendCauseMessage() ) {
+      if( cause != null && cause.getMessage() != null ) {
+        //Add the cause message to buffer
+        message.append(". Cause : " + cause.toString());
+      }
+    }
+
+    return message.toString();
+  }
+
+
+  /**
+   * Is this a retriable error.
+   * @return is it retriable
+   */
+  public boolean isRetriable() {
+    return errorType.isRetriable();
+  }
+
+  /**
+   * Gets the error type.
+   * @return the error type enum
+   */
+  public ErrorType getErrorType() {
+    return errorType;
+  }
+
+  /**
+   * Gets the error code.
+   * @return the error code
+   */
+  public int getErrorCode() {
+    return errorType.getErrorCode();
+  }
+
+  /* (non-Javadoc)
+   * @see java.lang.Throwable#toString()
+   */
+  @Override
+  public String toString() {
+    return getMessage();
+  }
+
+}

Added: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatUtil.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatUtil.java?rev=1091509&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatUtil.java (added)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatUtil.java Tue Apr 12 17:30:08 2011
@@ -0,0 +1,249 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hcatalog.common;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hcatalog.data.schema.HCatFieldSchema;
+import org.apache.hcatalog.data.schema.HCatSchema;
+import org.apache.hcatalog.data.schema.HCatSchemaUtils;
+
+public class HCatUtil {
+
+  public static boolean checkJobContextIfRunningFromBackend(JobContext j){
+    if (j.getConfiguration().get("mapred.task.id", "").equals("")){
+      return false;
+    }
+    return true;
+  }
+
+  public static String serialize(Serializable obj) throws IOException {
+    if (obj == null) {
+      return "";
+    }
+    try {
+      ByteArrayOutputStream serialObj = new ByteArrayOutputStream();
+      ObjectOutputStream objStream = new ObjectOutputStream(serialObj);
+      objStream.writeObject(obj);
+      objStream.close();
+      return encodeBytes(serialObj.toByteArray());
+    } catch (Exception e) {
+      throw new IOException("Serialization error: " + e.getMessage(), e);
+    }
+  }
+
+  public static Object deserialize(String str) throws IOException {
+    if (str == null || str.length() == 0) {
+      return null;
+    }
+    try {
+      ByteArrayInputStream serialObj = new ByteArrayInputStream(decodeBytes(str));
+      ObjectInputStream objStream = new ObjectInputStream(serialObj);
+      return objStream.readObject();
+    } catch (Exception e) {
+      throw new IOException("Deserialization error: " + e.getMessage(), e);
+    }
+  }
+
+  public static String encodeBytes(byte[] bytes) {
+    StringBuffer strBuf = new StringBuffer();
+
+    for (int i = 0; i < bytes.length; i++) {
+      strBuf.append((char) (((bytes[i] >> 4) & 0xF) + ('a')));
+      strBuf.append((char) (((bytes[i]) & 0xF) + ('a')));
+    }
+
+    return strBuf.toString();
+  }
+
+  public static byte[] decodeBytes(String str) {
+    byte[] bytes = new byte[str.length() / 2];
+    for (int i = 0; i < str.length(); i+=2) {
+      char c = str.charAt(i);
+      bytes[i/2] = (byte) ((c - 'a') << 4);
+      c = str.charAt(i+1);
+      bytes[i/2] += (c - 'a');
+    }
+    return bytes;
+  }
+
+  public static List<HCatFieldSchema> getHCatFieldSchemaList(List<FieldSchema> fields) throws HCatException {
+      if(fields == null) {
+          return null;
+      } else {
+          List<HCatFieldSchema> result = new ArrayList<HCatFieldSchema>();
+          for(FieldSchema f: fields) {
+              result.add(HCatSchemaUtils.getHCatFieldSchema(f));
+          }
+          return result;
+      }
+  }
+
+
+  public static HCatSchema extractSchemaFromStorageDescriptor(StorageDescriptor sd) throws HCatException {
+      if (sd == null){
+          throw new HCatException("Cannot construct partition info from an empty storage descriptor.");
+        }
+        HCatSchema schema = new HCatSchema(HCatUtil.getHCatFieldSchemaList(sd.getCols()));
+        return schema;
+  }
+
+  public static List<FieldSchema> getFieldSchemaList(List<HCatFieldSchema> howlFields) {
+      if(howlFields == null) {
+          return null;
+      } else {
+          List<FieldSchema> result = new ArrayList<FieldSchema>();
+          for(HCatFieldSchema f: howlFields) {
+              result.add(HCatSchemaUtils.getFieldSchema(f));
+          }
+          return result;
+      }
+  }
+
+  public static Table getTable(HiveMetaStoreClient client, String dbName, String tableName) throws Exception{
+    return client.getTable(dbName,tableName);
+  }
+
+  public static HCatSchema getTableSchemaWithPtnCols(Table table) throws IOException{
+      HCatSchema tableSchema = extractSchemaFromStorageDescriptor(table.getSd());
+
+      if( table.getPartitionKeys().size() != 0 ) {
+
+        // add partition keys to table schema
+        // NOTE : this assumes that we do not ever have ptn keys as columns inside the table schema as well!
+        for (FieldSchema fs : table.getPartitionKeys()){
+            tableSchema.append(HCatSchemaUtils.getHCatFieldSchema(fs));
+        }
+      }
+      return tableSchema;
+    }
+
+  /**
+   * Validate partition schema, checks if the column types match between the partition
+   * and the existing table schema. Returns the list of columns present in the partition
+   * but not in the table.
+   * @param table the table
+   * @param partitionSchema the partition schema
+   * @return the list of newly added fields
+   * @throws IOException Signals that an I/O exception has occurred.
+   */
+  public static List<FieldSchema> validatePartitionSchema(Table table, HCatSchema partitionSchema) throws IOException {
+    Map<String, FieldSchema> partitionKeyMap = new HashMap<String, FieldSchema>();
+
+    for(FieldSchema field : table.getPartitionKeys()) {
+      partitionKeyMap.put(field.getName().toLowerCase(), field);
+    }
+
+    List<FieldSchema> tableCols = table.getSd().getCols();
+    List<FieldSchema> newFields = new ArrayList<FieldSchema>();
+
+    for(int i = 0;i <  partitionSchema.getFields().size();i++) {
+
+      FieldSchema field = HCatSchemaUtils.getFieldSchema(partitionSchema.getFields().get(i));
+
+      FieldSchema tableField;
+      if( i < tableCols.size() ) {
+        tableField = tableCols.get(i);
+
+        if( ! tableField.getName().equalsIgnoreCase(field.getName())) {
+          throw new HCatException(ErrorType.ERROR_SCHEMA_COLUMN_MISMATCH, "Expected column <" + tableField.getName() +
+              "> at position " + (i + 1) + ", found column <" + field.getName() + ">");
+        }
+      } else {
+        tableField = partitionKeyMap.get(field.getName().toLowerCase());
+
+        if( tableField != null ) {
+          throw new HCatException(ErrorType.ERROR_SCHEMA_PARTITION_KEY, "Key <" +  field.getName() + ">");
+        }
+      }
+
+      if( tableField == null ) {
+        //field present in partition but not in table
+        newFields.add(field);
+      } else {
+        //field present in both. validate type has not changed
+        TypeInfo partitionType = TypeInfoUtils.getTypeInfoFromTypeString(field.getType());
+        TypeInfo tableType = TypeInfoUtils.getTypeInfoFromTypeString(tableField.getType());
+
+        if( ! partitionType.equals(tableType) ) {
+          throw new HCatException(ErrorType.ERROR_SCHEMA_TYPE_MISMATCH, "Column <" + field.getName() + ">, expected <" +
+              tableType.getTypeName() + ">, got <" + partitionType.getTypeName() + ">");
+        }
+      }
+    }
+
+    return newFields;
+  }
+
+  /**
+   * Test if the first FsAction is more permissive than the second. This is useful in cases where
+   * we want to ensure that a file owner has more permissions than the group they belong to, for eg.
+   * More completely(but potentially more cryptically)
+   *  owner-r >= group-r >= world-r : bitwise and-masked with 0444 => 444 >= 440 >= 400 >= 000
+   *  owner-w >= group-w >= world-w : bitwise and-masked with &0222 => 222 >= 220 >= 200 >= 000
+   *  owner-x >= group-x >= world-x : bitwise and-masked with &0111 => 111 >= 110 >= 100 >= 000
+   * @return true if first FsAction is more permissive than the second, false if not.
+   */
+  public static boolean validateMorePermissive(FsAction first, FsAction second) {
+    if ((first == FsAction.ALL) ||
+        (second == FsAction.NONE) ||
+        (first == second)) {
+      return true;
+    }
+    switch (first){
+      case READ_EXECUTE : return ((second == FsAction.READ) || (second == FsAction.EXECUTE));
+      case READ_WRITE : return ((second == FsAction.READ) || (second == FsAction.WRITE));
+      case WRITE_EXECUTE : return ((second == FsAction.WRITE) || (second == FsAction.EXECUTE));
+    }
+    return false;
+  }
+
+  /**
+   * Ensure that read or write permissions are not granted without also granting execute permissions.
+   * Essentially, r-- , rw- and -w- are invalid,
+   * r-x, -wx, rwx, ---, --x are valid
+   *
+   * @param perms The FsAction to verify
+   * @return true if the presence of read or write permission is accompanied by execute permissions
+   */
+  public static boolean validateExecuteBitPresentIfReadOrWrite(FsAction perms){
+    if ((perms == FsAction.READ) || (perms == FsAction.WRITE) || (perms == FsAction.READ_WRITE)){
+      return false;
+    }
+    return true;
+  }
+
+}

Added: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/DataType.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/DataType.java?rev=1091509&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/DataType.java (added)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/DataType.java Tue Apr 12 17:30:08 2011
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hcatalog.data;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.Map.Entry;
+
+
+public abstract class DataType {
+
+  public static final byte NULL      =   1;
+  public static final byte BOOLEAN   =   5;
+  public static final byte BYTE      =   6;
+  public static final byte INTEGER   =  10;
+  public static final byte SHORT     =  11;
+  public static final byte LONG      =  15;
+  public static final byte FLOAT     =  20;
+  public static final byte DOUBLE    =  25;
+  public static final byte STRING    =  55;
+
+  public static final byte MAP       = 100;
+  public static final byte STRUCT    = 110;
+  public static final byte LIST      = 120;
+  public static final byte ERROR     =  -1;
+
+  /**
+   * Determine the datatype of an object.
+   * @param o Object to test.
+   * @return byte code of the type, or ERROR if we don't know.
+   */
+  public static byte findType(Object o) {
+    if (o == null) {
+      return NULL;
+    }
+
+    Class<?> clazz = o.getClass();
+
+    // Try to put the most common first
+    if (clazz == String.class) {
+      return STRING;
+    } else if (clazz == Integer.class) {
+      return INTEGER;
+    } else if (clazz == Long.class) {
+      return LONG;
+    } else if (clazz == Float.class) {
+      return FLOAT;
+    } else if (clazz == Double.class) {
+      return DOUBLE;
+    } else if (clazz == Boolean.class) {
+      return BOOLEAN;
+    } else if (clazz == Byte.class) {
+      return BYTE;
+    } else if (clazz == Short.class) {
+      return SHORT;
+    } else if (o instanceof List<?>) {
+      return LIST;
+    } else if (o instanceof Map<?,?>) {
+      return MAP;
+    } else {return ERROR;}
+  }
+
+  public static int compare(Object o1, Object o2) {
+
+    return compare(o1, o2, findType(o1),findType(o2));
+  }
+
+  public static int compare(Object o1, Object o2, byte dt1, byte dt2) {
+    if (dt1 == dt2) {
+      switch (dt1) {
+      case NULL:
+        return 0;
+
+      case BOOLEAN:
+        return ((Boolean)o1).compareTo((Boolean)o2);
+
+      case BYTE:
+        return ((Byte)o1).compareTo((Byte)o2);
+
+      case INTEGER:
+        return ((Integer)o1).compareTo((Integer)o2);
+
+      case LONG:
+        return ((Long)o1).compareTo((Long)o2);
+
+      case FLOAT:
+        return ((Float)o1).compareTo((Float)o2);
+
+      case DOUBLE:
+        return ((Double)o1).compareTo((Double)o2);
+
+      case STRING:
+        return ((String)o1).compareTo((String)o2);
+
+      case SHORT:
+        return ((Short)o1).compareTo((Short)o2);
+
+      case LIST:
+        List<?> l1 = (List<?>)o1;
+        List<?> l2 = (List<?>)o2;
+        int len = l1.size();
+        if(len != l2.size()) {
+          return len - l2.size();
+        } else{
+          for(int i =0; i < len; i++){
+            int cmpVal = compare(l1.get(i), l2.get(i));
+            if(cmpVal != 0) {
+              return cmpVal;
+            }
+          }
+          return 0;
+        }
+
+      case MAP: {
+        Map<?,?> m1 = (Map<?,?>)o1;
+        Map<?,?> m2 = (Map<?,?>)o2;
+        int sz1 = m1.size();
+        int sz2 = m2.size();
+        if (sz1 < sz2) {
+          return -1;
+        } else if (sz1 > sz2) {
+          return 1;
+        } else {
+          // This is bad, but we have to sort the keys of the maps in order
+          // to be commutative.
+          TreeMap<Object,Object> tm1 = new TreeMap<Object,Object>(m1);
+          TreeMap<Object, Object> tm2 = new TreeMap<Object,Object>(m2);
+          Iterator<Entry<Object, Object>> i1 = tm1.entrySet().iterator();
+          Iterator<Entry<Object, Object> > i2 = tm2.entrySet().iterator();
+          while (i1.hasNext()) {
+            Map.Entry<Object, Object> entry1 = i1.next();
+            Map.Entry<Object, Object> entry2 = i2.next();
+            int c = compare(entry1.getValue(), entry2.getValue());
+            if (c != 0) {
+              return c;
+            } else {
+              c = compare(entry1.getValue(), entry2.getValue());
+              if (c != 0) {
+                return c;
+              }
+            }
+          }
+          return 0;
+        }
+      }
+
+      default:
+        throw new RuntimeException("Unkown type " + dt1 +
+        " in compare");
+      }
+    } else {
+      return dt1 < dt2 ? -1 : 1;
+    }
+  }
+}
\ No newline at end of file

Added: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/DefaultHCatRecord.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/DefaultHCatRecord.java?rev=1091509&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/DefaultHCatRecord.java (added)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/DefaultHCatRecord.java Tue Apr 12 17:30:08 2011
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hcatalog.data;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hcatalog.common.HCatException;
+import org.apache.hcatalog.data.schema.HCatSchema;
+
+public class DefaultHCatRecord extends HCatRecord {
+
+    private final List<Object> contents;
+
+    public DefaultHCatRecord(){
+        contents = new ArrayList<Object>();
+    }
+
+    public DefaultHCatRecord(int size){
+        contents = new ArrayList<Object>(size);
+        for(int i=0; i < size; i++){
+            contents.add(null);
+        }
+    }
+
+    @Override
+    public void remove(int idx) throws HCatException {
+      contents.remove(idx);
+    }
+
+    public DefaultHCatRecord(List<Object> list) {
+        contents = list;
+    }
+
+    @Override
+    public Object get(int fieldNum) {
+        return contents.get(fieldNum);
+    }
+
+    @Override
+    public List<Object> getAll() {
+        return contents;
+    }
+
+    @Override
+    public void set(int fieldNum, Object val) {
+        contents.set(fieldNum, val);
+    }
+
+    @Override
+    public int size() {
+        return contents.size();
+    }
+
+    @Override
+    public void readFields(DataInput in) throws IOException {
+
+        contents.clear();
+        int len = in.readInt();
+        for(int i =0; i < len; i++){
+            contents.add(ReaderWriter.readDatum(in));
+        }
+    }
+
+    @Override
+    public void write(DataOutput out) throws IOException {
+        int sz = size();
+        out.writeInt(sz);
+        for (int i = 0; i < sz; i++) {
+            ReaderWriter.writeDatum(out, contents.get(i));
+        }
+
+    }
+
+    @Override
+    public int compareTo(Object that) {
+
+        if(that instanceof HCatRecord) {
+            HCatRecord other = (HCatRecord)that;
+            int mySz = this.size();
+            int urSz = other.size();
+            if(mySz != urSz) {
+                return mySz - urSz;
+            } else{
+                for (int i = 0; i < mySz;i++) {
+                    int c = DataType.compare(get(i), other.get(i));
+                    if (c != 0) {
+                        return c;
+                    }
+                }
+            }
+            return 0;
+        } else {
+            return DataType.compare(this, that);
+        }
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        return (compareTo(other) == 0);
+    }
+
+    @Override
+    public int hashCode() {
+        int hash = 1;
+        for (Object o : contents) {
+            if (o != null) {
+                hash = 31 * hash + o.hashCode();
+            }
+        }
+        return hash;
+    }
+
+    @Override
+    public String toString() {
+
+        StringBuilder sb = new StringBuilder();
+        for(Object o : contents) {
+            sb.append(o+"\t");
+        }
+        return sb.toString();
+    }
+
+    @Override
+    public Object get(String fieldName, HCatSchema recordSchema) throws HCatException {
+        return get(recordSchema.getPosition(fieldName));
+    }
+
+    @Override
+    public void set(String fieldName, HCatSchema recordSchema, Object value) throws HCatException {
+        set(recordSchema.getPosition(fieldName),value);
+    }
+
+}

Added: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatArrayBag.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatArrayBag.java?rev=1091509&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatArrayBag.java (added)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatArrayBag.java Tue Apr 12 17:30:08 2011
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hcatalog.data;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.DefaultBagFactory;
+import org.apache.pig.data.DefaultDataBag;
+import org.apache.pig.data.DefaultTuple;
+import org.apache.pig.data.Tuple;
+
+public class HCatArrayBag<T> implements DataBag {
+
+  private static final long DUMMY_SIZE = 40;
+  List<T>  rawItemList = null;
+  DataBag convertedBag = null;
+//  List<Tuple> tupleList = null;
+
+  public class HowlArrayBagIterator implements Iterator<Tuple> {
+
+    Iterator<T> iter = null;
+
+    public HowlArrayBagIterator(List<T> rawItemList) {
+      iter = rawItemList.iterator();
+    }
+
+    @Override
+    public boolean hasNext() {
+      return iter.hasNext();
+    }
+
+    @Override
+    public Tuple next() {
+      Tuple t = new DefaultTuple();
+      t.append(iter.next());
+      return t;
+    }
+
+    @Override
+    public void remove() {
+      iter.remove();
+    }
+
+  }
+
+  public HCatArrayBag(List<T> list) {
+    rawItemList = list;
+  }
+
+  private void convertFromRawToTupleForm(){
+    if (convertedBag == null){
+      List<Tuple> ltuples = new ArrayList<Tuple>();
+      for (T item : rawItemList){
+        Tuple t = new DefaultTuple();
+        t.append(item);
+        ltuples.add(t);
+      }
+      convertedBag = DefaultBagFactory.getInstance().newDefaultBag(ltuples);
+    }else{
+      // TODO : throw exception or be silent? Currently going with silence, but needs revisiting.
+    }
+  }
+
+  @Override
+  public void add(Tuple t) {
+    if (convertedBag == null){
+      convertFromRawToTupleForm();
+    }
+    convertedBag.add(t);
+  }
+
+  @Override
+  public void addAll(DataBag db) {
+    Tuple t;
+    for (Iterator<Tuple> dbi = db.iterator() ; dbi.hasNext();){
+      this.add(dbi.next());
+    }
+  }
+
+  @Override
+  public void clear() {
+    rawItemList = null;
+    if (convertedBag != null){
+      convertedBag.clear();
+      convertedBag = null;
+    }
+  }
+
+  @Override
+  public boolean isDistinct() {
+    return false;
+  }
+
+  @Override
+  public boolean isSorted() {
+    return false;
+  }
+
+  @Override
+  public Iterator<Tuple> iterator() {
+    if (convertedBag != null){
+      return convertedBag.iterator();
+    }else{
+      return new HowlArrayBagIterator(rawItemList);
+    }
+  }
+
+  @Override
+  public void markStale(boolean arg0) {
+    // TODO Auto-generated method stub
+
+  }
+
+  @Override
+  public long size() {
+    return (convertedBag == null ? (rawItemList == null ? 0 : rawItemList.size()) : convertedBag.size() );
+  }
+
+  @Override
+  public long getMemorySize() {
+    // FIXME: put in actual impl
+    if (convertedBag != null){
+      return convertedBag.getMemorySize() + DUMMY_SIZE;
+    }else {
+      return DUMMY_SIZE;
+    }
+  }
+
+  @Override
+  public long spill() {
+    // FIXME: put in actual spill impl even for the list case
+    if (convertedBag != null){
+      return convertedBag.spill();
+    }
+    return 0;
+  }
+
+  @Override
+  public void readFields(DataInput arg0) throws IOException {
+    convertedBag = new DefaultDataBag();
+    convertedBag.readFields(arg0);
+  }
+
+  @Override
+  public void write(DataOutput arg0) throws IOException {
+    convertFromRawToTupleForm();
+    convertedBag.write(arg0);
+  }
+
+  @Override
+  public int compareTo(Object arg0) {
+    // TODO Auto-generated method stub - really need to put in a better implementation here, also, equality case not considered yet
+    return arg0.hashCode() < this.hashCode() ? -1 : 1;
+  }
+
+}

Added: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecord.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecord.java?rev=1091509&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecord.java (added)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecord.java Tue Apr 12 17:30:08 2011
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hcatalog.data;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hcatalog.common.HCatException;
+import org.apache.hcatalog.data.schema.HCatSchema;
+
+/**
+ * Abstract class exposing get and set semantics for basic record usage.
+ * Note :
+ *   HowlRecord is designed only to be used as in-memory representation only.
+ *   Don't use it to store data on the physical device.
+ */
+public abstract class HCatRecord implements HCatRecordable {
+
+    public abstract Object get(String fieldName, HCatSchema recordSchema) throws HCatException;
+    public abstract void set(String fieldName, HCatSchema recordSchema, Object value ) throws HCatException;
+    public abstract void remove(int idx) throws HCatException;
+
+    protected Object get(String fieldName, HCatSchema recordSchema, Class clazz) throws HCatException{
+        // TODO : if needed, verify that recordschema entry for fieldname matches appropriate type.
+        return get(fieldName,recordSchema);
+    }
+
+    public Boolean getBoolean(String fieldName, HCatSchema recordSchema) throws HCatException {
+        return (Boolean) get(fieldName, recordSchema, Boolean.class);
+    }
+
+    public void setBoolean(String fieldName, HCatSchema recordSchema, Boolean value) throws HCatException {
+        set(fieldName,recordSchema,value);
+    }
+
+    public Byte getByte(String fieldName, HCatSchema recordSchema) throws HCatException {
+        //TINYINT
+        return (Byte) get(fieldName, recordSchema, Byte.class);
+    }
+
+    public void setByte(String fieldName, HCatSchema recordSchema, Byte value) throws HCatException {
+        set(fieldName,recordSchema,value);
+    }
+
+    public Short getShort(String fieldName, HCatSchema recordSchema) throws HCatException {
+        // SMALLINT
+        return (Short) get(fieldName, recordSchema, Short.class);
+    }
+
+    public void setShort(String fieldName, HCatSchema recordSchema, Short value) throws HCatException {
+        set(fieldName,recordSchema,value);
+    }
+
+    public Integer getInteger(String fieldName, HCatSchema recordSchema) throws HCatException {
+        return (Integer) get(fieldName,recordSchema, Integer.class);
+    }
+
+    public void setInteger(String fieldName, HCatSchema recordSchema, Integer value) throws HCatException {
+        set(fieldName,recordSchema,value);
+    }
+
+    public Long getLong(String fieldName, HCatSchema recordSchema) throws HCatException {
+        // BIGINT
+        return (Long) get(fieldName,recordSchema,Long.class);
+    }
+
+    public void setLong(String fieldName, HCatSchema recordSchema, Long value) throws HCatException {
+        set(fieldName,recordSchema,value);
+    }
+
+    public Float getFloat(String fieldName, HCatSchema recordSchema) throws HCatException {
+        return (Float) get(fieldName,recordSchema,Float.class);
+    }
+
+    public void setFloat(String fieldName, HCatSchema recordSchema, Float value) throws HCatException {
+        set(fieldName,recordSchema,value);
+    }
+
+    public Double getDouble(String fieldName, HCatSchema recordSchema) throws HCatException {
+        return (Double) get(fieldName,recordSchema,Double.class);
+    }
+
+    public void setDouble(String fieldName, HCatSchema recordSchema, Double value) throws HCatException {
+        set(fieldName,recordSchema,value);
+    }
+
+    public String getString(String fieldName, HCatSchema recordSchema) throws HCatException {
+        return (String) get(fieldName,recordSchema,String.class);
+    }
+
+    public void setString(String fieldName, HCatSchema recordSchema, String value) throws HCatException {
+        set(fieldName,recordSchema,value);
+    }
+
+    @SuppressWarnings("unchecked")
+    public List<? extends Object> getStruct(String fieldName, HCatSchema recordSchema) throws HCatException {
+        return (List<? extends Object>) get(fieldName,recordSchema,List.class);
+    }
+
+    public void setStruct(String fieldName, HCatSchema recordSchema, List<? extends Object> value) throws HCatException {
+        set(fieldName,recordSchema,value);
+    }
+
+    public List<?> getList(String fieldName, HCatSchema recordSchema) throws HCatException {
+        return (List<?>) get(fieldName,recordSchema,List.class);
+    }
+
+    public void setList(String fieldName, HCatSchema recordSchema, List<?> value) throws HCatException {
+        set(fieldName,recordSchema,value);
+    }
+
+    public Map<?,?> getMap(String fieldName, HCatSchema recordSchema) throws HCatException {
+        return (Map<?,?>) get(fieldName,recordSchema,Map.class);
+    }
+
+    public void setMap(String fieldName, HCatSchema recordSchema, Map<?,?> value) throws HCatException {
+        set(fieldName,recordSchema,value);
+    }
+
+}

Added: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecordable.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecordable.java?rev=1091509&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecordable.java (added)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecordable.java Tue Apr 12 17:30:08 2011
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.data;
+
+import java.util.List;
+
+import org.apache.hadoop.io.WritableComparable;
+
+/**
+ * Interface that determines whether we can implement a HowlRecord on top of it
+ */
+public interface HCatRecordable extends WritableComparable<Object> {
+
+  /**
+   * Gets the field at the specified index.
+   * @param fieldNum the field number
+   * @return the object at the specified index
+   */
+  Object get(int fieldNum);
+
+  /**
+   * Gets all the fields of the howl record.
+   * @return the list of fields
+   */
+  List<Object> getAll();
+
+  /**
+   * Sets the field at the specified index.
+   * @param fieldNum the field number
+   * @param value the value to set
+   */
+  void set(int fieldNum, Object value);
+
+  /**
+   * Gets the size of the howl record.
+   * @return the size
+   */
+  int size();
+
+}

Added: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/Pair.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/Pair.java?rev=1091509&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/Pair.java (added)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/Pair.java Tue Apr 12 17:30:08 2011
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.data;
+
+import java.io.Serializable;
+
+/**
+ * Copy of C++ STL pair container.
+ */
+public class Pair<T, U> implements Serializable {
+
+    private static final long serialVersionUID = 1L;
+    public T first;
+    public U second;
+
+    /**
+     * @param f First element in pair.
+     * @param s Second element in pair.
+     */
+    public Pair(T f, U s) {
+        first = f;
+        second = s;
+    }
+
+    /* (non-Javadoc)
+     * @see java.lang.Object#toString()
+     */
+    @Override
+    public String toString() {
+        return "[" + first.toString() +"," + second.toString() + "]";
+    }
+
+    @Override
+    public int hashCode() {
+        return (((this.first == null ? 1 : this.first.hashCode()) * 17)
+                + (this.second == null ? 1 : this.second.hashCode()) * 19);
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        if(other == null) {
+            return false;
+        }
+
+        if(! (other instanceof Pair)) {
+            return false;
+        }
+
+        Pair otherPair = (Pair) other;
+
+        if(this.first == null) {
+            if(otherPair.first != null) {
+                return false;
+            } else {
+                return true;
+            }
+        }
+
+        if(this.second == null) {
+            if(otherPair.second != null) {
+                return false;
+            } else {
+                return true;
+            }
+        }
+
+        if(this.first.equals(otherPair.first) && this.second.equals(otherPair.second)) {
+            return true;
+        } else {
+            return false;
+        }
+    }
+}

Added: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/ReaderWriter.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/ReaderWriter.java?rev=1091509&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/ReaderWriter.java (added)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/ReaderWriter.java Tue Apr 12 17:30:08 2011
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hcatalog.data;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.hadoop.io.VIntWritable;
+import org.apache.hadoop.io.VLongWritable;
+
+
+public abstract class ReaderWriter {
+
+  private static final String UTF8 = "UTF-8";
+
+  public static Object readDatum(DataInput in) throws IOException {
+
+    byte type = in.readByte();
+    switch (type) {
+
+    case DataType.STRING:
+      byte[] buffer = new byte[in.readInt()];
+      in.readFully(buffer);
+      return new String(buffer,UTF8);
+
+    case DataType.INTEGER:
+      VIntWritable vint = new VIntWritable();
+      vint.readFields(in);
+      return vint.get();
+
+    case DataType.LONG:
+      VLongWritable vlong = new VLongWritable();
+      vlong.readFields(in);
+      return vlong.get();
+
+    case DataType.FLOAT:
+      return in.readFloat();
+
+    case DataType.DOUBLE:
+      return in.readDouble();
+
+    case DataType.BOOLEAN:
+      return in.readBoolean();
+
+    case DataType.BYTE:
+      return in.readByte();
+
+    case DataType.SHORT:
+      return in.readShort();
+
+    case DataType.NULL:
+      return null;
+
+    case DataType.MAP:
+      int size = in.readInt();
+      Map<Object,Object> m = new HashMap<Object, Object>(size);
+      for (int i = 0; i < size; i++) {
+          m.put(readDatum(in), readDatum(in));
+      }
+      return m;
+
+    case DataType.LIST:
+      int sz = in.readInt();
+      List<Object> list = new ArrayList<Object>(sz);
+      for(int i=0; i < sz; i++) {
+        list.add(readDatum(in));
+      }
+      return list;
+
+    default:
+      throw new IOException("Unexpected data type " + type +
+          " found in stream.");
+    }
+  }
+
+  public static void writeDatum(DataOutput out, Object val) throws IOException {
+    // write the data type
+    byte type = DataType.findType(val);
+    switch (type) {
+    case DataType.LIST:
+      out.writeByte(DataType.LIST);
+      List<?> list = (List<?>)val;
+      int sz = list.size();
+      out.writeInt(sz);
+      for (int i = 0; i < sz; i++) {
+        writeDatum(out, list.get(i));
+      }
+      return;
+
+    case DataType.MAP:
+      out.writeByte(DataType.MAP);
+      Map<?,?> m = (Map<?, ?>)val;
+      out.writeInt(m.size());
+      Iterator<?> i =
+        m.entrySet().iterator();
+      while (i.hasNext()) {
+        Entry<?,?> entry = (Entry<?, ?>) i.next();
+        writeDatum(out, entry.getKey());
+        writeDatum(out, entry.getValue());
+      }
+      return;
+
+    case DataType.INTEGER:
+      out.writeByte(DataType.INTEGER);
+      new VIntWritable((Integer)val).write(out);
+      return;
+
+    case DataType.LONG:
+      out.writeByte(DataType.LONG);
+      new VLongWritable((Long)val).write(out);
+      return;
+
+    case DataType.FLOAT:
+      out.writeByte(DataType.FLOAT);
+      out.writeFloat((Float)val);
+      return;
+
+    case DataType.DOUBLE:
+      out.writeByte(DataType.DOUBLE);
+      out.writeDouble((Double)val);
+      return;
+
+    case DataType.BOOLEAN:
+      out.writeByte(DataType.BOOLEAN);
+      out.writeBoolean((Boolean)val);
+      return;
+
+    case DataType.BYTE:
+      out.writeByte(DataType.BYTE);
+      out.writeByte((Byte)val);
+      return;
+
+    case DataType.SHORT:
+      out.writeByte(DataType.SHORT);
+      out.writeShort((Short)val);
+      return;
+
+    case DataType.STRING:
+      String s = (String)val;
+      byte[] utfBytes = s.getBytes(ReaderWriter.UTF8);
+      out.writeByte(DataType.STRING);
+      out.writeInt(utfBytes.length);
+      out.write(utfBytes);
+      return;
+
+
+    case DataType.NULL:
+      out.writeByte(DataType.NULL);
+      return;
+
+    default:
+      throw new IOException("Unexpected data type " + type +
+          " found in stream.");
+    }
+  }
+}

Added: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatFieldSchema.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatFieldSchema.java?rev=1091509&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatFieldSchema.java (added)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatFieldSchema.java Tue Apr 12 17:30:08 2011
@@ -0,0 +1,239 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.data.schema;
+
+import java.io.Serializable;
+
+import org.apache.hcatalog.common.HCatException;
+
+public class HCatFieldSchema implements Serializable {
+
+    public enum Type {
+        INT,
+        TINYINT,
+        SMALLINT,
+        BIGINT,
+        BOOLEAN,
+        FLOAT,
+        DOUBLE,
+        STRING,
+        ARRAY,
+        MAP,
+        STRUCT,
+    }
+
+    public enum Category {
+        PRIMITIVE,
+        ARRAY,
+        MAP,
+        STRUCT;
+
+        public static Category fromType(Type type) {
+            if (Type.ARRAY == type){
+                return ARRAY;
+            }else if(Type.STRUCT == type){
+                return STRUCT;
+            }else if (Type.MAP == type){
+                return MAP;
+            }else{
+                return PRIMITIVE;
+            }
+        }
+    };
+
+    public boolean isComplex(){
+      return (category == Category.PRIMITIVE) ? false : true;
+    }
+
+    /**
+     *
+     */
+    private static final long serialVersionUID = 1L;
+
+    String fieldName = null;
+    String comment = null;
+    Type type = null;
+    Category category = null;
+
+    // Populated if column is struct, array or map types.
+    // If struct type, contains schema of the struct.
+    // If array type, contains schema of one of the elements.
+    // If map type, contains schema of the value element.
+    HCatSchema subSchema = null;
+
+    // populated if column is Map type
+    Type mapKeyType = null;
+
+    private String typeString = null;
+
+    @SuppressWarnings("unused")
+    private HCatFieldSchema(){
+        // preventing empty ctor from being callable
+    }
+
+    /**
+     * Returns type of the field
+     * @return type of the field
+     */
+    public Type getType(){
+        return type;
+    }
+
+    /**
+     * Returns category of the field
+     * @return category of the field
+     */
+    public Category getCategory(){
+        return category;
+    }
+
+    /**
+     * Returns name of the field
+     * @return name of the field
+     */
+    public String getName(){
+        return fieldName;
+    }
+
+    public String getComment(){
+        return comment;
+    }
+
+    /**
+     * Constructor constructing a primitive datatype HCatFieldSchema
+     * @param fieldName Name of the primitive field
+     * @param type Type of the primitive field
+     * @throws HCatException if call made on non-primitive types
+     */
+    public HCatFieldSchema(String fieldName, Type type, String comment) throws HCatException {
+        assertTypeInCategory(type,Category.PRIMITIVE);
+        this.fieldName = fieldName;
+        this.type = type;
+        this.category = Category.PRIMITIVE;
+        this.comment = comment;
+    }
+
+    /**
+     * Constructor for constructing a ARRAY type or STRUCT type HCatFieldSchema, passing type and subschema
+     * @param fieldName Name of the array or struct field
+     * @param type Type of the field - either Type.ARRAY or Type.STRUCT
+     * @param subSchema - subschema of the struct, or element schema of the elements in the array
+     * @throws HCatException if call made on Primitive or Map types
+     */
+    public HCatFieldSchema(String fieldName, Type type, HCatSchema subSchema,String comment) throws HCatException{
+        assertTypeNotInCategory(type,Category.PRIMITIVE);
+        assertTypeNotInCategory(type,Category.MAP);
+        this.fieldName = fieldName;
+        this.type = type;
+        this.category = Category.fromType(type);
+        this.subSchema = subSchema;
+        if(type == Type.ARRAY){
+         this.subSchema.get(0).setName(null);
+        }
+        this.comment = comment;
+    }
+
+    private void setName(String name) {
+      this.fieldName = name;
+    }
+
+    /**
+     * Constructor for constructing a MAP type HCatFieldSchema, passing type of key and value
+     * @param fieldName Name of the array or struct field
+     * @param type Type of the field - must be Type.MAP
+     * @param mapKeyType - key type of the Map
+     * @param mapValueSchema - subschema of the value of the Map
+     * @throws HCatException if call made on non-Map types
+     */
+    public HCatFieldSchema(String fieldName, Type type, Type mapKeyType, HCatSchema mapValueSchema, String comment) throws HCatException{
+        assertTypeInCategory(type,Category.MAP);
+        assertTypeInCategory(mapKeyType,Category.PRIMITIVE);
+        this.fieldName = fieldName;
+        this.type = Type.MAP;
+        this.category = Category.MAP;
+        this.mapKeyType = mapKeyType;
+        this.subSchema = mapValueSchema;
+        this.subSchema.get(0).setName(null);
+        this.comment = comment;
+    }
+
+    public HCatSchema getStructSubSchema() throws HCatException {
+        assertTypeInCategory(this.type,Category.STRUCT);
+        return subSchema;
+    }
+
+    public HCatSchema getArrayElementSchema() throws HCatException {
+        assertTypeInCategory(this.type,Category.ARRAY);
+        return subSchema;
+    }
+
+    public Type getMapKeyType() throws HCatException {
+        assertTypeInCategory(this.type,Category.MAP);
+        return mapKeyType;
+    }
+
+    public HCatSchema getMapValueSchema() throws HCatException {
+        assertTypeInCategory(this.type,Category.MAP);
+        return subSchema;
+    }
+
+    private static void assertTypeInCategory(Type type, Category category) throws HCatException {
+        Category typeCategory = Category.fromType(type);
+        if (typeCategory != category){
+            throw new HCatException("Type category mismatch. Expected "+category+" but type "+type+" in category "+typeCategory);
+        }
+    }
+
+    private static void assertTypeNotInCategory(Type type, Category category) throws HCatException {
+        Category typeCategory = Category.fromType(type);
+        if (typeCategory == category){
+            throw new HCatException("Type category mismatch. Expected type "+type+" not in category "+category+" but was so.");
+        }
+    }
+
+    @Override
+    public String toString(){
+        return getTypeString();
+    }
+
+    public String getTypeString(){
+        if (typeString != null){
+            return typeString;
+        }
+
+        StringBuilder sb = new StringBuilder();
+        if (Category.PRIMITIVE == category){
+            sb.append(type);
+        }else if (Category.STRUCT == category){
+            sb.append("struct<");
+            sb.append(subSchema.toString());
+            sb.append(">");
+        }else if (Category.ARRAY == category){
+            sb.append("array<");
+            sb.append(subSchema.toString());
+            sb.append(">");
+        }else if (Category.MAP == category){
+            sb.append("map<");
+            sb.append(mapKeyType);
+            sb.append(",");
+            sb.append(subSchema.toString());
+            sb.append(">");
+        }
+        return (typeString = sb.toString().toLowerCase());
+    }
+}

Added: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatSchema.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatSchema.java?rev=1091509&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatSchema.java (added)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatSchema.java Tue Apr 12 17:30:08 2011
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.data.schema;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hcatalog.common.HCatException;
+
+/**
+ * HCatSchema. This class is NOT thread-safe.
+ */
+
+public class HCatSchema implements Serializable{
+
+    private static final long serialVersionUID = 1L;
+
+    private final List<HCatFieldSchema> fieldSchemas;
+    private final Map<String,Integer> fieldPositionMap;
+    private final List<String> fieldNames;
+
+    /**
+     *
+     * @param fieldSchemas is now owned by HCatSchema. Any subsequent modifications
+     * on fieldSchemas won't get reflected in HCatSchema.
+     */
+    public HCatSchema(final List<HCatFieldSchema> fieldSchemas){
+        this.fieldSchemas = new ArrayList<HCatFieldSchema>(fieldSchemas);
+        int idx = 0;
+        fieldPositionMap = new HashMap<String,Integer>();
+        fieldNames = new ArrayList<String>();
+        for (HCatFieldSchema field : fieldSchemas){
+            fieldPositionMap.put(field.getName(), idx);
+            fieldNames.add(field.getName());
+            idx++;
+        }
+    }
+
+    public void append(final HCatFieldSchema hfs) throws HCatException{
+
+      if(hfs == null || fieldSchemas == null){
+        throw new HCatException("Attempt to append null HCatFieldSchema in HCatSchema.");
+      }
+      //TODO Addition of existing field should not be allowed in Schema.
+      //Need to enforce that. For that to happen, field schema needs to implement Comparable.
+      // Also, HCatSchema needs to implement Comparable.
+
+      this.fieldSchemas.add(hfs);
+      String fieldName = hfs.getName();
+      this.fieldNames.add(fieldName);
+      this.fieldPositionMap.put(fieldName, this.size()-1);
+    }
+
+    /**
+     *  Users are not allowed to modify the list directly, since HCatSchema
+     *  maintains internal state. Use append/remove to modify the schema.
+     */
+    public List<HCatFieldSchema> getFields(){
+        return Collections.unmodifiableList(this.fieldSchemas);
+    }
+
+    /**
+     * @param fieldName
+     * @return the index of field named fieldName in Schema. If field is not
+     * present, returns null.
+     */
+    public Integer getPosition(String fieldName) {
+      return fieldPositionMap.get(fieldName);
+    }
+
+    public HCatFieldSchema get(String fieldName) throws HCatException {
+        return get(getPosition(fieldName));
+    }
+
+    public List<String> getFieldNames(){
+        return this.fieldNames;
+    }
+
+    public HCatFieldSchema get(int position) {
+        return fieldSchemas.get(position);
+    }
+
+    public int size(){
+      return fieldSchemas.size();
+    }
+
+    public void remove(final HCatFieldSchema howlFieldSchema) throws HCatException {
+
+      if(!fieldSchemas.contains(howlFieldSchema)){
+        throw new HCatException("Attempt to delete a non-existent column from Howl Schema: "+ howlFieldSchema);
+      }
+
+      fieldSchemas.remove(howlFieldSchema);
+      fieldPositionMap.remove(howlFieldSchema);
+      fieldNames.remove(howlFieldSchema.getName());
+    }
+
+    @Override
+    public String toString() {
+        boolean first = true;
+        StringBuilder sb = new StringBuilder();
+        for (HCatFieldSchema hfs : fieldSchemas){
+            if (!first){
+                sb.append(",");
+            }else{
+                first = false;
+            }
+            if (hfs.getName() != null){
+                sb.append(hfs.getName());
+                sb.append(":");
+            }
+            sb.append(hfs.toString());
+        }
+        return sb.toString();
+    }
+}

Added: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatSchemaUtils.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatSchemaUtils.java?rev=1091509&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatSchemaUtils.java (added)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatSchemaUtils.java Tue Apr 12 17:30:08 2011
@@ -0,0 +1,223 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.data.schema;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.Schema;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
+import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+import org.apache.hcatalog.common.HCatException;
+import org.apache.hcatalog.data.schema.HCatFieldSchema.Type;
+
+
+public class HCatSchemaUtils {
+
+    private static HCatSchemaUtils ref = new HCatSchemaUtils();
+
+    public static CollectionBuilder getStructSchemaBuilder(){
+        return ref.new CollectionBuilder();
+    }
+
+    public static CollectionBuilder getListSchemaBuilder(){
+        return ref.new CollectionBuilder();
+    }
+
+    public static MapBuilder getMapSchemaBuilder(){
+        return ref.new MapBuilder();
+    }
+
+
+    public abstract class HCatSchemaBuilder {
+        public abstract HCatSchema build() throws HCatException;
+    }
+
+    public class CollectionBuilder extends HCatSchemaBuilder { // for STRUCTS(multiple-add-calls) and LISTS(single-add-call)
+        List<HCatFieldSchema> fieldSchemas = null;
+
+        CollectionBuilder(){
+            fieldSchemas = new ArrayList<HCatFieldSchema>();
+        }
+
+        public CollectionBuilder addField(FieldSchema fieldSchema) throws HCatException{
+            return this.addField(getHCatFieldSchema(fieldSchema));
+        }
+
+        public CollectionBuilder addField(HCatFieldSchema fieldColumnSchema){
+            fieldSchemas.add(fieldColumnSchema);
+            return this;
+        }
+
+        @Override
+        public HCatSchema build() throws HCatException{
+            return new HCatSchema(fieldSchemas);
+        }
+
+    }
+
+    public class MapBuilder extends HCatSchemaBuilder {
+
+        Type keyType = null;
+        HCatSchema valueSchema = null;
+
+        @Override
+        public HCatSchema build() throws HCatException {
+            List<HCatFieldSchema> fslist = new ArrayList<HCatFieldSchema>();
+            fslist.add(new HCatFieldSchema(null,Type.MAP,keyType,valueSchema,null));
+            return new HCatSchema(fslist);
+        }
+
+        public MapBuilder withValueSchema(HCatSchema valueSchema) {
+            this.valueSchema = valueSchema;
+            return this;
+        }
+
+        public MapBuilder withKeyType(Type keyType) {
+            this.keyType = keyType;
+            return this;
+        }
+
+    }
+
+
+    /**
+     * Convert a HCatFieldSchema to a FieldSchema
+     * @param fs FieldSchema to convert
+     * @return HCatFieldSchema representation of FieldSchema
+     * @throws HCatException
+     */
+    public static HCatFieldSchema getHCatFieldSchema(FieldSchema fs) throws HCatException {
+        String fieldName = fs.getName();
+        TypeInfo baseTypeInfo = TypeInfoUtils.getTypeInfoFromTypeString(fs.getType());
+        return getHCatFieldSchema(fieldName, baseTypeInfo);
+    }
+
+    private static HCatFieldSchema getHCatFieldSchema(String fieldName, TypeInfo fieldTypeInfo) throws HCatException {
+        Category typeCategory = fieldTypeInfo.getCategory();
+        if (Category.PRIMITIVE == typeCategory){
+            return new HCatFieldSchema(fieldName,getPrimitiveHType(fieldTypeInfo),null);
+        } else if (Category.STRUCT == typeCategory) {
+            HCatSchema subSchema = constructHCatSchema((StructTypeInfo)fieldTypeInfo);
+            return new HCatFieldSchema(fieldName,HCatFieldSchema.Type.STRUCT,subSchema,null);
+        } else if (Category.LIST == typeCategory) {
+            HCatSchema subSchema = getHCatSchema(((ListTypeInfo)fieldTypeInfo).getListElementTypeInfo());
+            return new HCatFieldSchema(fieldName,HCatFieldSchema.Type.ARRAY,subSchema,null);
+        } else if (Category.MAP == typeCategory) {
+            HCatFieldSchema.Type mapKeyType =  getPrimitiveHType(((MapTypeInfo)fieldTypeInfo).getMapKeyTypeInfo());
+            HCatSchema subSchema = getHCatSchema(((MapTypeInfo)fieldTypeInfo).getMapValueTypeInfo());
+            return new HCatFieldSchema(fieldName,HCatFieldSchema.Type.MAP,mapKeyType,subSchema,null);
+        } else{
+            throw new TypeNotPresentException(fieldTypeInfo.getTypeName(),null);
+        }
+    }
+
+    private static Type getPrimitiveHType(TypeInfo basePrimitiveTypeInfo) {
+        switch(((PrimitiveTypeInfo)basePrimitiveTypeInfo).getPrimitiveCategory()) {
+        case BOOLEAN:
+            return Type.BOOLEAN;
+        case BYTE:
+            return Type.TINYINT;
+        case DOUBLE:
+            return Type.DOUBLE;
+        case FLOAT:
+            return Type.FLOAT;
+        case INT:
+            return Type.INT;
+        case LONG:
+            return Type.BIGINT;
+        case SHORT:
+            return Type.SMALLINT;
+        case STRING:
+            return Type.STRING;
+        default:
+            throw new TypeNotPresentException(((PrimitiveTypeInfo)basePrimitiveTypeInfo).getTypeName(), null);
+        }
+    }
+
+    public static HCatSchema getHCatSchema(Schema schema) throws HCatException{
+        return getHCatSchema(schema.getFieldSchemas());
+    }
+
+    public static HCatSchema getHCatSchema(List<? extends FieldSchema> fslist) throws HCatException{
+        CollectionBuilder builder = getStructSchemaBuilder();
+        for (FieldSchema fieldSchema : fslist){
+            builder.addField(fieldSchema);
+        }
+        return builder.build();
+    }
+
+    private static HCatSchema constructHCatSchema(StructTypeInfo stypeInfo) throws HCatException {
+        CollectionBuilder builder = getStructSchemaBuilder();
+        for (String fieldName : ((StructTypeInfo)stypeInfo).getAllStructFieldNames()){
+            builder.addField(getHCatFieldSchema(fieldName,((StructTypeInfo)stypeInfo).getStructFieldTypeInfo(fieldName)));
+        }
+        return builder.build();
+    }
+
+    public static HCatSchema getHCatSchema(TypeInfo typeInfo) throws HCatException {
+        Category typeCategory = typeInfo.getCategory();
+        if (Category.PRIMITIVE == typeCategory){
+            return getStructSchemaBuilder().addField(new HCatFieldSchema(null,getPrimitiveHType(typeInfo),null)).build();
+        } else if (Category.STRUCT == typeCategory) {
+            HCatSchema subSchema = constructHCatSchema((StructTypeInfo) typeInfo);
+            return getStructSchemaBuilder().addField(new HCatFieldSchema(null,Type.STRUCT,subSchema,null)).build();
+        } else if (Category.LIST == typeCategory) {
+            CollectionBuilder builder = getStructSchemaBuilder();
+            builder.addField(getHCatFieldSchema(null,((ListTypeInfo)typeInfo).getListElementTypeInfo()));
+            return builder.build();
+        } else if (Category.MAP == typeCategory) {
+            HCatFieldSchema.Type mapKeyType =  getPrimitiveHType(((MapTypeInfo)typeInfo).getMapKeyTypeInfo());
+            HCatSchema subSchema = getHCatSchema(((MapTypeInfo)typeInfo).getMapValueTypeInfo());
+            MapBuilder builder = getMapSchemaBuilder();
+            return builder.withKeyType(mapKeyType).withValueSchema(subSchema).build();
+        } else{
+            throw new TypeNotPresentException(typeInfo.getTypeName(),null);
+        }
+    }
+
+    public static HCatSchema getHCatSchemaFromTypeString(String typeString) throws HCatException {
+        return getHCatSchema(TypeInfoUtils.getTypeInfoFromTypeString(typeString));
+    }
+
+    public static HCatSchema getHCatSchema(String schemaString) throws HCatException {
+        if ((schemaString == null) || (schemaString.trim().isEmpty())){
+            return new HCatSchema(new ArrayList<HCatFieldSchema>()); // empty HSchema construct
+        }
+        HCatSchema outerSchema = getHCatSchemaFromTypeString("struct<"+schemaString+">");
+        return outerSchema.get(0).getStructSubSchema();
+    }
+
+    public static FieldSchema getFieldSchema(HCatFieldSchema howlFieldSchema){
+        return new FieldSchema(howlFieldSchema.getName(),howlFieldSchema.getTypeString(),howlFieldSchema.getComment());
+    }
+
+    public static List<FieldSchema> getFieldSchemas(List<HCatFieldSchema> howlFieldSchemas){
+        List<FieldSchema> lfs = new ArrayList<FieldSchema>();
+        for (HCatFieldSchema hfs : howlFieldSchemas){
+            lfs.add(getFieldSchema(hfs));
+        }
+        return lfs;
+    }
+}



Mime
View raw message