hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From na...@apache.org
Subject svn commit: r922404 [3/3] - in /hadoop/hive/trunk: ./ hbase-handler/ hbase-handler/lib/ hbase-handler/src/ hbase-handler/src/java/ hbase-handler/src/java/org/ hbase-handler/src/java/org/apache/ hbase-handler/src/java/org/apache/hadoop/ hbase-handler/sr...
Date Fri, 12 Mar 2010 19:53:59 GMT
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java?rev=922404&r1=922403&r2=922404&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java Fri Mar 12 19:53:57 2010
@@ -36,6 +36,7 @@ import org.apache.hadoop.hive.ql.io.Hive
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
 import org.apache.hadoop.hive.ql.plan.FileSinkDesc;
+import org.apache.hadoop.hive.ql.plan.PlanUtils;
 import org.apache.hadoop.hive.ql.plan.api.OperatorType;
 import org.apache.hadoop.hive.serde2.SerDeException;
 import org.apache.hadoop.hive.serde2.Serializer;
@@ -70,6 +71,7 @@ public class FileSinkOperator extends Te
   protected transient BytesWritable commonKey = new BytesWritable();
   protected transient TableIdEnum tabIdEnum = null;
   private transient LongWritable row_count;
+  private transient boolean isNativeTable = true;
 
   /**
    * The evaluators for the multiFile sprayer. If the table under consideration has 1000 buckets,
@@ -114,8 +116,11 @@ public class FileSinkOperator extends Te
   protected transient boolean autoDelete = false;
 
   private void commit(int idx) throws IOException {
-    if (!fs.rename(outPaths[idx], finalPaths[idx])) {
-      throw new IOException("Unable to rename output to: " + finalPaths[idx]);
+    if (isNativeTable) {
+      if (!fs.rename(outPaths[idx], finalPaths[idx])) {
+        throw new IOException("Unable to rename output to: " 
+          + finalPaths[idx]);
+      }
     }
     LOG.info("Committed " + outPaths[idx] + " to output file: " + finalPaths[idx]);
   }
@@ -126,6 +131,7 @@ public class FileSinkOperator extends Te
       serializer = (Serializer) conf.getTableInfo().getDeserializerClass()
           .newInstance();
       serializer.initialize(null, conf.getTableInfo().getProperties());
+      isNativeTable = !conf.getTableInfo().isNonNative();
 
       JobConf jc;
       if (hconf instanceof JobConf) {
@@ -158,7 +164,6 @@ public class FileSinkOperator extends Te
       Path tmpPath = Utilities.toTempPath(specPath);
       Set<Integer> seenBuckets = new HashSet<Integer>();
       fs = (new Path(specPath)).getFileSystem(hconf);
-
       HiveOutputFormat<?, ?> hiveOutputFormat = conf.getTableInfo()
         .getOutputFileFormatClass().newInstance();
       boolean isCompressed = conf.getCompressed();
@@ -193,12 +198,14 @@ public class FileSinkOperator extends Te
           taskId = Utilities.replaceTaskIdFromFilename(Utilities.getTaskId(hconf), bucketNum);
         }
 
-        finalPaths[filesIdx] = new Path(tmpPath, taskId);
-        LOG.info("Final Path: FS " + finalPaths[filesIdx]);
-
-        outPaths[filesIdx] = new Path(tmpPath, Utilities.toTempPath(taskId));
-
-        LOG.info("Writing to temp file: FS " + outPaths[filesIdx]);
+        if (isNativeTable) {
+          finalPaths[filesIdx] = new Path(tmpPath, taskId);
+          LOG.info("Final Path: FS " + finalPaths[filesIdx]);
+          outPaths[filesIdx] = new Path(tmpPath, Utilities.toTempPath(taskId));
+          LOG.info("Writing to temp file: FS " + outPaths[filesIdx]);
+        } else {
+          finalPaths[filesIdx] = outPaths[filesIdx] = new Path(specPath);
+        }
 
         // The reason to keep these instead of using
         // OutputFormat.getRecordWriter() is that
@@ -208,6 +215,7 @@ public class FileSinkOperator extends Te
                                                                             hiveOutputFormat, isCompressed, finalPaths[filesIdx]);
         LOG.info("New Final Path: FS " + finalPaths[filesIdx]);
 
+        Utilities.copyTableJobPropertiesToConf(conf.getTableInfo(), jc);
         outWriters[filesIdx] = HiveFileFormatUtils.getHiveRecordWriter(jc, conf
                                                                        .getTableInfo(), outputClass, conf, outPaths[filesIdx]);
 
@@ -217,7 +225,10 @@ public class FileSinkOperator extends Te
       assert filesIdx == numFiles;
 
       // in recent hadoop versions, use deleteOnExit to clean tmp files.
-      autoDelete = ShimLoader.getHadoopShims().fileSystemDeleteOnExit(fs, outPaths[0]);
+      if (isNativeTable) {
+        autoDelete = ShimLoader.getHadoopShims().fileSystemDeleteOnExit(fs,
+          outPaths[0]);
+      }
 
       int id = conf.getDestTableId();
       if ((id != 0) && (id <= TableIdEnum.values().length)) {
@@ -303,7 +314,7 @@ public class FileSinkOperator extends Te
       try {
         for (int idx = 0; idx < numFiles; idx++) {
           outWriters[idx].close(abort);
-          if (!autoDelete) {
+          if (!autoDelete && isNativeTable) {
             fs.delete(outPaths[idx], true);
           }
         }
@@ -324,7 +335,7 @@ public class FileSinkOperator extends Te
   @Override
   public void jobClose(Configuration hconf, boolean success) throws HiveException {
     try {
-      if (conf != null) {
+      if ((conf != null) && isNativeTable) {
         String specPath = conf.getDirName();
         FileSinkOperator.mvFileToFinalPath(specPath, hconf, success, LOG);
       }
@@ -363,4 +374,10 @@ public class FileSinkOperator extends Te
   public int getType() {
     return OperatorType.FILESINK;
   }
+
+  @Override
+  public void augmentPlan() {
+    PlanUtils.configureTableJobPropertiesForStorageHandler(
+      getConf().getTableInfo());
+  }
 }

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java?rev=922404&r1=922403&r2=922404&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java Fri Mar 12 19:53:57 2010
@@ -1145,6 +1145,14 @@ public abstract class Operator<T extends
     return groupKeyObject;
   }
 
+  /**
+   * Called during semantic analysis as operators are being added
+   * in order to give them a chance to compute any additional plan information
+   * needed.  Does nothing by default.
+   */
+  public void augmentPlan() {
+  }
+
   public ExecMapperContext getExecContext() {
     return execContext;
   }

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java?rev=922404&r1=922403&r2=922404&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java Fri Mar 12 19:53:57 2010
@@ -1197,4 +1197,22 @@ public final class Utilities {
     }
     return notificationInterval;
   }
+
+  /**
+   * Copies the storage handler properties configured for a table descriptor
+   * to a runtime job configuration.
+   *
+   * @param tbl table descriptor from which to read
+   *
+   * @param job configuration which receives configured properties
+   */
+  public static void copyTableJobPropertiesToConf(TableDesc tbl, JobConf job) {
+    Map<String, String> jobProperties = tbl.getJobProperties();
+    if (jobProperties == null) {
+      return;
+    }
+    for (Map.Entry<String, String> entry : jobProperties.entrySet()) {
+      job.set(entry.getKey(), entry.getValue());
+    }
+  }
 }

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java?rev=922404&r1=922403&r2=922404&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java Fri Mar 12 19:53:57 2010
@@ -219,6 +219,13 @@ public class HiveInputFormat<K extends W
 
     InputFormat inputFormat = getInputFormatFromCache(inputFormatClass,
         cloneJobConf);
+
+    Path [] paths = FileInputFormat.getInputPaths(job);
+    // for now we only get one path for splits which access a non-native
+    // table; should probably add a corresponding assertion
+    PartitionDesc part = getPartitionDescFromPath(
+      pathToPartitionInfo, paths[0]);
+    Utilities.copyTableJobPropertiesToConf(part.getTableDesc(), cloneJobConf);
     return new HiveRecordReader(inputFormat.getRecordReader(inputSplit,
         cloneJobConf, reporter));
   }
@@ -249,6 +256,7 @@ public class HiveInputFormat<K extends W
       // class
       Class inputFormatClass = part.getInputFileFormatClass();
       InputFormat inputFormat = getInputFormatFromCache(inputFormatClass, job);
+      Utilities.copyTableJobPropertiesToConf(part.getTableDesc(), newjob);
 
       FileInputFormat.setInputPaths(newjob, dir);
       newjob.setInputFormat(inputFormat.getClass());

Added: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/DefaultStorageHandler.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/DefaultStorageHandler.java?rev=922404&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/DefaultStorageHandler.java (added)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/DefaultStorageHandler.java Fri Mar 12 19:53:57 2010
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.metadata;
+
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.HiveMetaHook;
+import org.apache.hadoop.hive.ql.plan.TableDesc;
+import org.apache.hadoop.hive.serde2.SerDe;
+import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe;
+import org.apache.hadoop.mapred.InputFormat;
+import org.apache.hadoop.mapred.OutputFormat;
+import org.apache.hadoop.mapred.SequenceFileInputFormat;
+import org.apache.hadoop.mapred.SequenceFileOutputFormat;
+
+/**
+ * DefaultStorageHandler is an implementation of {@link HiveStorageHandler}
+ * which supplies the standard defaults for all options.  This can be useful
+ * either as the base class for custom storage handlers, or as a mock in tests
+ * (providing something which appears to be a non-native table with respect to
+ * metadata even though its behavior is otherwise identical to a native table).
+ */
+public class DefaultStorageHandler implements HiveStorageHandler {
+  private Configuration conf;
+
+  @Override
+  public Class<? extends InputFormat> getInputFormatClass() {
+    return SequenceFileInputFormat.class;
+  }
+
+  @Override
+  public Class<? extends OutputFormat> getOutputFormatClass() {
+    return SequenceFileOutputFormat.class;
+  }
+
+  @Override
+  public Class<? extends SerDe> getSerDeClass() {
+    return LazySimpleSerDe.class;
+  }
+
+  @Override
+  public HiveMetaHook getMetaHook() {
+    // no hook by default
+    return null;
+  }
+
+  @Override
+  public void configureTableJobProperties(
+    TableDesc tableDesc,
+    Map<String, String> jobProperties) {
+    // do nothing by default
+  }
+
+  @Override
+  public Configuration getConf() {
+    return conf;
+  }
+
+  @Override
+  public void setConf(Configuration conf) {
+    this.conf = conf;
+  }
+}

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java?rev=922404&r1=922403&r2=922404&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java Fri Mar 12 19:53:57 2010
@@ -28,18 +28,21 @@ import java.util.Properties;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.common.JavaUtils;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.HiveMetaException;
+import org.apache.hadoop.hive.metastore.HiveMetaHook;
+import org.apache.hadoop.hive.metastore.HiveMetaHookLoader;
 import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
 import org.apache.hadoop.hive.metastore.IMetaStoreClient;
 import org.apache.hadoop.hive.metastore.MetaStoreUtils;
 import org.apache.hadoop.hive.metastore.TableType;
 import org.apache.hadoop.hive.metastore.Warehouse;
 import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
+import org.apache.hadoop.hive.metastore.api.Constants;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
 import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
 import org.apache.hadoop.hive.metastore.api.MetaException;
@@ -53,6 +56,7 @@ import org.apache.hadoop.hive.serde2.laz
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.io.WritableComparable;
 import org.apache.hadoop.mapred.InputFormat;
+import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.thrift.TException;
 
@@ -918,7 +922,32 @@ public class Hive {
    *           if a working client can't be created
    */
   private IMetaStoreClient createMetaStoreClient() throws MetaException {
-    return new HiveMetaStoreClient(conf);
+
+    HiveMetaHookLoader hookLoader = new HiveMetaHookLoader() {
+        public HiveMetaHook getHook(
+          org.apache.hadoop.hive.metastore.api.Table tbl)
+          throws MetaException {
+
+          try {
+            if (tbl == null) {
+              return null;
+            }
+            HiveStorageHandler storageHandler =
+              HiveUtils.getStorageHandler(
+                conf,
+                tbl.getParameters().get(Constants.META_TABLE_STORAGE));
+            if (storageHandler == null) {
+              return null;
+            }
+            return storageHandler.getMetaHook();
+          } catch (HiveException ex) {
+            LOG.error(StringUtils.stringifyException(ex));
+            throw new MetaException(
+              "Failed to load storage handler:  " + ex.getMessage());
+          }
+        }
+      };
+    return new HiveMetaStoreClient(conf, hookLoader);
   }
 
   /**

Added: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java?rev=922404&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java (added)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java Fri Mar 12 19:53:57 2010
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.metadata;
+
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.hive.metastore.HiveMetaHook;
+import org.apache.hadoop.hive.ql.plan.TableDesc;
+import org.apache.hadoop.hive.serde2.SerDe;
+import org.apache.hadoop.mapred.InputFormat;
+import org.apache.hadoop.mapred.OutputFormat;
+
+/**
+ * HiveStorageHandler defines a pluggable interface for adding
+ * new storage handlers to Hive.  A storage handler consists of
+ * a bundle of the following:
+ *
+ *<ul>
+ *<li>input format
+ *<li>output format
+ *<li>serde
+ *<li>metadata hooks for keeping an external catalog in sync
+ * with Hive's metastore
+ *<li>rules for setting up the configuration properties on
+ * map/reduce jobs which access tables stored by this handler
+ *</ul>
+ *
+ * Storage handler classes are plugged in using the STORED BY 'classname'
+ * clause in CREATE TABLE.
+ */
+public interface HiveStorageHandler extends Configurable {
+  /**
+   * @return Class providing an implementation of {@link InputFormat}
+   */
+  public Class<? extends InputFormat> getInputFormatClass();
+
+  /**
+   * @return Class providing an implementation of {@link OutputFormat}
+   */
+  public Class<? extends OutputFormat> getOutputFormatClass();
+
+  /**
+   * @return Class providing an implementation of {@link SerDe}
+   */
+  public Class<? extends SerDe> getSerDeClass();
+
+  /**
+   * @return metadata hook implementation, or null if this
+   * storage handler does not need any metadata notifications
+   */
+  public HiveMetaHook getMetaHook();
+
+  /**
+   * Configures properties for a job based on the definition of the
+   * source or target table it accesses.
+   *
+   * @param tableDesc descriptor for the table being accessed
+   *
+   * @param jobProperties receives properties copied or transformed
+   * from the table properties
+   */
+  public void configureTableJobProperties(
+    TableDesc tableDesc,
+    Map<String, String> jobProperties);
+}

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveUtils.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveUtils.java?rev=922404&r1=922403&r2=922404&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveUtils.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveUtils.java Fri Mar 12 19:53:57 2010
@@ -18,6 +18,10 @@
 
 package org.apache.hadoop.hive.ql.metadata;
 
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.JavaUtils;
+import org.apache.hadoop.util.ReflectionUtils;
+
 /**
  * General collection of helper functions.
  * 
@@ -120,6 +124,25 @@ public final class HiveUtils {
     return "`" + identifier + "`";
   }
 
+  public static HiveStorageHandler getStorageHandler(
+    Configuration conf, String className) throws HiveException {
+    
+    if (className == null) {
+      return null;
+    }
+    try {
+      Class<? extends HiveStorageHandler> handlerClass =
+        (Class<? extends HiveStorageHandler>)
+        Class.forName(className, true, JavaUtils.getClassLoader());
+      HiveStorageHandler storageHandler = (HiveStorageHandler)
+        ReflectionUtils.newInstance(handlerClass, conf);
+      return storageHandler;
+    } catch (ClassNotFoundException e) {
+      throw new HiveException("Error in loading storage handler."
+          + e.getMessage(), e);
+    }
+  }
+
   private HiveUtils() {
     // prevent instantiation
   }

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java?rev=922404&r1=922403&r2=922404&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java Fri Mar 12 19:53:57 2010
@@ -78,6 +78,7 @@ public class Table implements Serializab
   private Class<? extends HiveOutputFormat> outputFormatClass;
   private Class<? extends InputFormat> inputFormatClass;
   private URI uri;
+  private HiveStorageHandler storageHandler;
   
   /**
    * Used only for serialization.
@@ -223,7 +224,11 @@ public class Table implements Serializab
   }
 
   final public Path getPath() {
-    return new Path(tTable.getSd().getLocation());
+    String location = tTable.getSd().getLocation();
+    if (location == null) {
+      return null;
+    }
+    return new Path(location);
   }
 
   final public String getTableName() {
@@ -232,7 +237,10 @@ public class Table implements Serializab
 
   final public URI getDataLocation() {
     if (uri == null) {
-      uri = getPath().toUri();
+      Path path = getPath();
+      if (path != null) {
+        uri = path.toUri();
+      }
     }
     return uri;
   }
@@ -250,11 +258,34 @@ public class Table implements Serializab
     return deserializer;
   }
 
+  public HiveStorageHandler getStorageHandler() {
+    if (storageHandler != null) {
+      return storageHandler;
+    }
+    try {
+      storageHandler = HiveUtils.getStorageHandler(
+        Hive.get().getConf(),
+        getProperty(
+          org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_STORAGE));
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+    return storageHandler;
+  }
+
   final public Class<? extends InputFormat> getInputFormatClass() {
     if (inputFormatClass == null) {
       try {
-        inputFormatClass = (Class<? extends InputFormat>)
-            Class.forName(tTable.getSd().getInputFormat(), true, JavaUtils.getClassLoader());
+        String className = tTable.getSd().getInputFormat();
+        if (className == null) {
+          if (getStorageHandler() == null) {
+            return null;
+          }
+          inputFormatClass = getStorageHandler().getInputFormatClass();
+        } else {
+          inputFormatClass = (Class<? extends InputFormat>)
+            Class.forName(className, true, JavaUtils.getClassLoader());
+        }
       } catch (ClassNotFoundException e) {
         throw new RuntimeException(e);
       }
@@ -267,14 +298,22 @@ public class Table implements Serializab
     
     if (outputFormatClass == null) {
       try {
-        Class<?> c = Class.forName(tTable.getSd().getOutputFormat(), true, 
+        String className = tTable.getSd().getOutputFormat();
+        Class<?> c;
+        if (className == null) {
+          if (getStorageHandler() == null) {
+            return null;
+          }
+          c = getStorageHandler().getOutputFormatClass();
+        } else {
+          c = Class.forName(className, true,
             JavaUtils.getClassLoader());
+        }
         if (!HiveOutputFormat.class.isAssignableFrom(c)) {
           outputFormatClass = HiveFileFormatUtils.getOutputFormatSubstitute(c);
         } else {
           outputFormatClass = (Class<? extends HiveOutputFormat>)c;
         }
-            
       } catch (ClassNotFoundException e) {
         throw new RuntimeException(e);
       }
@@ -509,6 +548,11 @@ public class Table implements Serializab
   }
 
   public void setInputFormatClass(String name) throws HiveException {
+    if (name == null) {
+      inputFormatClass = null;
+      tTable.getSd().setInputFormat(null);
+      return;
+    }
     try {
       setInputFormatClass((Class<? extends InputFormat<WritableComparable, Writable>>) Class
           .forName(name, true, JavaUtils.getClassLoader()));
@@ -518,6 +562,11 @@ public class Table implements Serializab
   }
 
   public void setOutputFormatClass(String name) throws HiveException {
+    if (name == null) {
+      outputFormatClass = null;
+      tTable.getSd().setOutputFormat(null);
+      return;
+    }
     try {
       Class<?> origin = Class.forName(name, true, JavaUtils.getClassLoader());
       setOutputFormatClass(HiveFileFormatUtils
@@ -690,4 +739,10 @@ public class Table implements Serializab
   public void setCreateTime(int createTime) {
     tTable.setCreateTime(createTime);
   }
+
+  public boolean isNonNative() {
+    return getProperty(
+      org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_STORAGE)
+      != null;
+  }
 };

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java?rev=922404&r1=922403&r2=922404&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java Fri Mar 12 19:53:57 2010
@@ -21,6 +21,7 @@ package org.apache.hadoop.hive.ql.parse;
 import java.io.Serializable;
 import java.io.UnsupportedEncodingException;
 import java.util.ArrayList;
+import java.util.Map;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedHashMap;
@@ -211,6 +212,25 @@ public abstract class BaseSemanticAnalyz
     return val;
   }
 
+  /**
+   * Converts parsed key/value properties pairs into a map.
+   *
+   * @param prop ASTNode parent of the key/value pairs
+   *
+   * @param mapProp property map which receives the mappings
+   */
+  public static void readProps(
+    ASTNode prop, Map<String, String> mapProp) {
+
+    for (int propChild = 0; propChild < prop.getChildCount(); propChild++) {
+      String key = unescapeSQLString(prop.getChild(propChild).getChild(0)
+          .getText());
+      String value = unescapeSQLString(prop.getChild(propChild).getChild(1)
+          .getText());
+      mapProp.put(key, value);
+    }
+  }
+
   @SuppressWarnings("nls")
   public static String unescapeSQLString(String b) {
 

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java?rev=922404&r1=922403&r2=922404&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java Fri Mar 12 19:53:57 2010
@@ -196,6 +196,7 @@ public class DDLSemanticAnalyzer extends
     String tableName = unescapeIdentifier(ast.getChild(0).getText());
     String inputFormat = null;
     String outputFormat = null;
+    String storageHandler = null;
     String serde = null;
     ASTNode child = (ASTNode) ast.getChild(1);
 
@@ -212,6 +213,15 @@ public class DDLSemanticAnalyzer extends
         throw new SemanticException(e);
       }
       break;
+    case HiveParser.TOK_STORAGEHANDLER:
+      storageHandler =
+        unescapeSQLString(((ASTNode) child.getChild(1)).getToken().getText());
+      try {
+        Class.forName(storageHandler);
+      } catch (ClassNotFoundException e) {
+        throw new SemanticException(e);
+      }
+      break;
     case HiveParser.TOK_TBLSEQUENCEFILE:
       inputFormat = SEQUENCEFILE_INPUT;
       outputFormat = SEQUENCEFILE_OUTPUT;
@@ -227,7 +237,7 @@ public class DDLSemanticAnalyzer extends
       break;
     }
     AlterTableDesc alterTblDesc = new AlterTableDesc(tableName, inputFormat,
-        outputFormat, serde);
+        outputFormat, serde, storageHandler);
     rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(),
         alterTblDesc), conf));
   }
@@ -254,15 +264,9 @@ public class DDLSemanticAnalyzer extends
         alterTblDesc), conf));
   }
 
-  private HashMap<String, String> getProps(ASTNode prop) {
+  static HashMap<String, String> getProps(ASTNode prop) {
     HashMap<String, String> mapProp = new HashMap<String, String>();
-    for (int propChild = 0; propChild < prop.getChildCount(); propChild++) {
-      String key = unescapeSQLString(prop.getChild(propChild).getChild(0)
-          .getText());
-      String value = unescapeSQLString(prop.getChild(propChild).getChild(1)
-          .getText());
-      mapProp.put(key, value);
-    }
+    readProps(prop, mapProp);
     return mapProp;
   }
 

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java?rev=922404&r1=922403&r2=922404&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java Fri Mar 12 19:53:57 2010
@@ -138,7 +138,9 @@ public enum ErrorMsg {
       + "number of column names specified by CREATE VIEW"),
   DML_AGAINST_VIEW("A view cannot be used as target table for LOAD or INSERT"),
   UNSUPPORTED_TYPE("DATE, DATETIME, and TIMESTAMP types aren't supported yet. Please use "
-      + "STRING instead.");
+      + "STRING instead."),
+  CREATE_NON_NATIVE_AS("CREATE TABLE AS SELECT cannot be used for a non-native table"),
+  LOAD_INTO_NON_NATIVE("A non-native table cannot be used as target for LOAD");
 
   private String mesg;
   private String sqlState;

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g?rev=922404&r1=922403&r2=922404&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g Fri Mar 12 19:53:57 2010
@@ -119,6 +119,7 @@ TOK_TBLSEQUENCEFILE;
 TOK_TBLTEXTFILE;
 TOK_TBLRCFILE;
 TOK_TABLEFILEFORMAT;
+TOK_STORAGEHANDLER;
 TOK_ALTERTABLE_CLUSTER_SORT;
 TOK_TABCOLNAME;
 TOK_TABLELOCATION;
@@ -577,6 +578,9 @@ tableFileFormat
       | KW_STORED KW_AS KW_RCFILE  -> TOK_TBLRCFILE
       | KW_STORED KW_AS KW_INPUTFORMAT inFmt=StringLiteral KW_OUTPUTFORMAT outFmt=StringLiteral
       -> ^(TOK_TABLEFILEFORMAT $inFmt $outFmt)
+      | KW_STORED KW_BY storageHandler=StringLiteral
+         (KW_WITH KW_SERDEPROPERTIES serdeprops=tableProperties)?
+      -> ^(TOK_STORAGEHANDLER $storageHandler $serdeprops?)
     ;
 
 tableLocation

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java?rev=922404&r1=922403&r2=922404&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java Fri Mar 12 19:53:57 2010
@@ -194,6 +194,9 @@ public class LoadSemanticAnalyzer extend
     if (ts.tableHandle.isView()) {
       throw new SemanticException(ErrorMsg.DML_AGAINST_VIEW.getMsg());
     }
+    if (ts.tableHandle.isNonNative()) {
+      throw new SemanticException(ErrorMsg.LOAD_INTO_NON_NATIVE.getMsg());
+    }
     URI toURI = (ts.partHandle != null) ? ts.partHandle.getDataLocation()
         : ts.tableHandle.getDataLocation();
 

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=922404&r1=922403&r2=922404&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Fri Mar 12 19:53:57 2010
@@ -1129,6 +1129,7 @@ public class SemanticAnalyzer extends Ba
       RowResolver rr) {
     OpParseContext ctx = new OpParseContext(rr);
     opParseCtx.put(op, ctx);
+    op.augmentPlan();
     return op;
   }
 
@@ -3137,8 +3138,8 @@ public class SemanticAnalyzer extends Ba
     Integer dest_type = qbm.getDestTypeForAlias(dest);
 
     Table dest_tab; // destination table if any
-    String queryTmpdir; // the intermediate destination directory
-    Path dest_path; // the final destination directory
+    String queryTmpdir = null; // the intermediate destination directory
+    Path dest_path = null; // the final destination directory
     TableDesc table_desc = null;
     int currentTableId = 0;
     boolean isLocal = false;
@@ -3155,7 +3156,12 @@ public class SemanticAnalyzer extends Ba
         throw new SemanticException(ErrorMsg.NEED_PARTITION_ERROR.getMsg());
       }
       dest_path = dest_tab.getPath();
-      queryTmpdir = ctx.getExternalTmpFileURI(dest_path.toUri());
+      boolean isNonNativeTable = dest_tab.isNonNative();
+      if (isNonNativeTable) {
+        queryTmpdir = dest_path.toUri().getPath();
+      } else {
+        queryTmpdir = ctx.getExternalTmpFileURI(dest_path.toUri());
+      }
       table_desc = Utilities.getTableDesc(dest_tab);
 
       // Add sorting/bucketing if needed
@@ -3166,9 +3172,11 @@ public class SemanticAnalyzer extends Ba
       destTableId++;
 
       // Create the work for moving the table
-      loadTableWork.add(new LoadTableDesc(queryTmpdir, ctx
-          .getExternalTmpFileURI(dest_path.toUri()), table_desc,
-          new HashMap<String, String>()));
+      if (!isNonNativeTable) {
+        loadTableWork.add(new LoadTableDesc(queryTmpdir, ctx
+            .getExternalTmpFileURI(dest_path.toUri()), table_desc,
+            new HashMap<String, String>()));
+      }
       if (!outputs.add(new WriteEntity(dest_tab))) {
         throw new SemanticException(ErrorMsg.OUTPUT_SPECIFIED_MULTIPLE_TIMES
             .getMsg(dest_tab.getTableName()));
@@ -3627,7 +3635,7 @@ public class SemanticAnalyzer extends Ba
           column = null;
         } else {
           column = TypeCheckProcFactory.DefaultExprProcessor
-            .getFuncExprNodeDesc(tableFieldTypeInfo.getTypeName(), 
+            .getFuncExprNodeDesc(tableFieldTypeInfo.getTypeName(),
                 column);
         }
         if (column == null) {
@@ -5542,7 +5550,7 @@ public class SemanticAnalyzer extends Ba
 
             // If there is any unknown partition, create a map-reduce job for
             // the filter to prune correctly
-            if (partsList.getUnknownPartns().size() == 0) {
+            if ((partsList.getUnknownPartns().size() == 0)) {
               List<String> listP = new ArrayList<String>();
               List<PartitionDesc> partP = new ArrayList<PartitionDesc>();
 
@@ -5568,6 +5576,10 @@ public class SemanticAnalyzer extends Ba
       }
 
       if (noMapRed) {
+        if (fetch.getTblDesc() != null) {
+          PlanUtils.configureTableJobPropertiesForStorageHandler(
+            fetch.getTblDesc());
+        }
         fetchTask = (FetchTask) TaskFactory.get(fetch, conf);
         setFetchTask(fetchTask);
 
@@ -6235,6 +6247,7 @@ public class SemanticAnalyzer extends Ba
     String outputFormat = null;
     String location = null;
     String serde = null;
+    String storageHandler = null;
     Map<String, String> mapProp = new HashMap<String, String>();
     boolean ifNotExists = false;
     boolean isExt = false;
@@ -6348,19 +6361,12 @@ public class SemanticAnalyzer extends Ba
         }
         break;
       case HiveParser.TOK_TABLESERIALIZER:
-
         child = (ASTNode) child.getChild(0);
         serde = unescapeSQLString(child.getChild(0).getText());
         if (child.getChildCount() == 2) {
-          mapProp = new HashMap<String, String>();
-          ASTNode prop = (ASTNode) ((ASTNode) child.getChild(1)).getChild(0);
-          for (int propChild = 0; propChild < prop.getChildCount(); propChild++) {
-            String key = unescapeSQLString(prop.getChild(propChild).getChild(0)
-                .getText());
-            String value = unescapeSQLString(prop.getChild(propChild).getChild(
-                1).getText());
-            mapProp.put(key, value);
-          }
+          readProps(
+            (ASTNode) (child.getChild(1).getChild(0)),
+            mapProp);
         }
         break;
       case HiveParser.TOK_TBLSEQUENCEFILE:
@@ -6383,12 +6389,24 @@ public class SemanticAnalyzer extends Ba
       case HiveParser.TOK_TABLELOCATION:
         location = unescapeSQLString(child.getChild(0).getText());
         break;
+      case HiveParser.TOK_STORAGEHANDLER:
+        storageHandler = unescapeSQLString(child.getChild(0).getText());
+        if (child.getChildCount() == 2) {
+          readProps(
+            (ASTNode) (child.getChild(1).getChild(0)),
+            mapProp);
+        }
+        break;
       default:
         assert false;
       }
     }
 
-    if (inputFormat == null) {
+    if ((command_type == CTAS) && (storageHandler != null)) {
+      throw new SemanticException(ErrorMsg.CREATE_NON_NATIVE_AS.getMsg());
+    }
+
+    if ((inputFormat == null) && (storageHandler == null)) {
       assert outputFormat == null;
       if ("SequenceFile".equalsIgnoreCase(conf.getVar(HiveConf.ConfVars.HIVEDEFAULTFILEFORMAT))) {
         inputFormat = SEQUENCEFILE_INPUT;
@@ -6423,7 +6441,7 @@ public class SemanticAnalyzer extends Ba
       crtTblDesc = new CreateTableDesc(tableName, isExt, cols, partCols,
           bucketCols, sortCols, numBuckets, fieldDelim, fieldEscape,
           collItemDelim, mapKeyDelim, lineDelim, comment, inputFormat,
-          outputFormat, location, serde, mapProp, ifNotExists);
+          outputFormat, location, serde, storageHandler, mapProp, ifNotExists);
 
       validateCreateTable(crtTblDesc);
       rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(),
@@ -6455,7 +6473,7 @@ public class SemanticAnalyzer extends Ba
       crtTblDesc = new CreateTableDesc(tableName, isExt, cols, partCols,
           bucketCols, sortCols, numBuckets, fieldDelim, fieldEscape,
           collItemDelim, mapKeyDelim, lineDelim, comment, inputFormat,
-          outputFormat, location, serde, mapProp, ifNotExists);
+          outputFormat, location, serde, storageHandler, mapProp, ifNotExists);
       qb.setTableDesc(crtTblDesc);
 
       return selectStmt;
@@ -6540,17 +6558,19 @@ public class SemanticAnalyzer extends Ba
       return;
     }
 
-    try {
-      Class<?> origin = Class.forName(crtTblDesc.getOutputFormat(), true,
+    if (crtTblDesc.getStorageHandler() == null) {
+      try {
+        Class<?> origin = Class.forName(crtTblDesc.getOutputFormat(), true,
           JavaUtils.getClassLoader());
-      Class<? extends HiveOutputFormat> replaced = HiveFileFormatUtils
+        Class<? extends HiveOutputFormat> replaced = HiveFileFormatUtils
           .getOutputFormatSubstitute(origin);
-      if (replaced == null) {
-        throw new SemanticException(ErrorMsg.INVALID_OUTPUT_FORMAT_TYPE
+        if (replaced == null) {
+          throw new SemanticException(ErrorMsg.INVALID_OUTPUT_FORMAT_TYPE
             .getMsg());
+        }
+      } catch (ClassNotFoundException e) {
+        throw new SemanticException(ErrorMsg.INVALID_OUTPUT_FORMAT_TYPE.getMsg());
       }
-    } catch (ClassNotFoundException e) {
-      throw new SemanticException(ErrorMsg.INVALID_OUTPUT_FORMAT_TYPE.getMsg());
     }
 
     List<String> colNames = validateColumnNameUniqueness(crtTblDesc.getCols());

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java?rev=922404&r1=922403&r2=922404&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java Fri Mar 12 19:53:57 2010
@@ -52,6 +52,7 @@ public class AlterTableDesc extends DDLD
   HashMap<String, String> props;
   String inputFormat;
   String outputFormat;
+  String storageHandler;
   int numberBuckets;
   ArrayList<String> bucketColumns;
   ArrayList<Order> sortColumns;
@@ -132,13 +133,14 @@ public class AlterTableDesc extends DDLD
    *          new table output format
    */
   public AlterTableDesc(String name, String inputFormat, String outputFormat,
-      String serdeName) {
+      String serdeName, String storageHandler) {
     super();
     op = AlterTableTypes.ADDFILEFORMAT;
     oldName = name;
     this.inputFormat = inputFormat;
     this.outputFormat = outputFormat;
     this.serdeName = serdeName;
+    this.storageHandler = storageHandler;
   }
 
   public AlterTableDesc(String tableName, int numBuckets,
@@ -296,6 +298,22 @@ public class AlterTableDesc extends DDLD
   }
 
   /**
+   * @return the storage handler
+   */
+  @Explain(displayName = "storage handler")
+  public String getStorageHandler() {
+    return storageHandler;
+  }
+
+  /**
+   * @param storageHandler
+   *          the storage handler to set
+   */
+  public void setStorageHandler(String storageHandler) {
+    this.storageHandler = storageHandler;
+  }
+
+  /**
    * @return the number of buckets
    */
   public int getNumberBuckets() {

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java?rev=922404&r1=922403&r2=922404&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java Fri Mar 12 19:53:57 2010
@@ -52,6 +52,7 @@ public class CreateTableDesc extends DDL
   String outputFormat;
   String location;
   String serName;
+  String storageHandler;
   HashMap<String, String> mapProp;
   boolean ifNotExists;
 
@@ -64,6 +65,7 @@ public class CreateTableDesc extends DDL
       String fieldDelim, String fieldEscape, String collItemDelim,
       String mapKeyDelim, String lineDelim, String comment, String inputFormat,
       String outputFormat, String location, String serName,
+      String storageHandler,
       Map<String, String> mapProp, boolean ifNotExists) {
     this.tableName = tableName;
     this.isExternal = isExternal;
@@ -82,6 +84,7 @@ public class CreateTableDesc extends DDL
     this.numBuckets = numBuckets;
     this.partCols = new ArrayList<FieldSchema>(partCols);
     this.serName = serName;
+    this.storageHandler = storageHandler;
     this.mapProp = new HashMap<String, String>(mapProp);
     this.ifNotExists = ifNotExists;
   }
@@ -220,6 +223,15 @@ public class CreateTableDesc extends DDL
     this.outputFormat = outputFormat;
   }
 
+  @Explain(displayName = "storage handler")
+  public String getStorageHandler() {
+    return storageHandler;
+  }
+
+  public void setStorageHandler(String storageHandler) {
+    this.storageHandler = storageHandler;
+  }
+
   @Explain(displayName = "location")
   public String getLocation() {
     return location;

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MapredLocalWork.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MapredLocalWork.java?rev=922404&r1=922403&r2=922404&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MapredLocalWork.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MapredLocalWork.java Fri Mar 12 19:53:57 2010
@@ -41,7 +41,7 @@ public class MapredLocalWork implements 
   private LinkedHashMap<String, FetchWork> aliasToFetchWork;
   private boolean inputFileChangeSensitive;
   private BucketMapJoinContext bucketMapjoinContext;
-  
+
   public MapredLocalWork() {
   }
 
@@ -86,13 +86,20 @@ public class MapredLocalWork implements 
   public void setInputFileChangeSensitive(boolean inputFileChangeSensitive) {
     this.inputFileChangeSensitive = inputFileChangeSensitive;
   }
-  
+
   public void deriveExplainAttributes() {
     if (bucketMapjoinContext != null) {
       bucketMapjoinContext.deriveBucketMapJoinMapping();
     }
+    for (FetchWork fetchWork : aliasToFetchWork.values()) {
+      if (fetchWork.getTblDesc() == null) {
+        continue;
+      }
+      PlanUtils.configureTableJobPropertiesForStorageHandler(
+        fetchWork.getTblDesc());
+    }
   }
-  
+
   @Explain(displayName = "Bucket Mapjoin Context", normalExplain = false)
   public BucketMapJoinContext getBucketMapjoinContext() {
     return bucketMapjoinContext;
@@ -101,32 +108,32 @@ public class MapredLocalWork implements 
   public void setBucketMapjoinContext(BucketMapJoinContext bucketMapjoinContext) {
     this.bucketMapjoinContext = bucketMapjoinContext;
   }
-  
+
   public static class BucketMapJoinContext implements Serializable {
-    
+
     private static final long serialVersionUID = 1L;
-    
+
     // used for bucket map join
     private LinkedHashMap<String, LinkedHashMap<String, ArrayList<String>>> aliasBucketFileNameMapping;
     private String mapJoinBigTableAlias;
     private Class<? extends BucketMatcher> bucketMatcherClass;
-    
+
     private LinkedHashMap<String, LinkedHashMap<String, ArrayList<String>>> aliasBucketBaseFileNameMapping;
 
     public void setMapJoinBigTableAlias(String bigTableAlias) {
       this.mapJoinBigTableAlias = bigTableAlias;
     }
 
-    
+
     public void deriveBucketMapJoinMapping() {
       if (aliasBucketFileNameMapping != null) {
-        Iterator<Entry<String, LinkedHashMap<String, ArrayList<String>>>> iter =  
+        Iterator<Entry<String, LinkedHashMap<String, ArrayList<String>>>> iter =
           aliasBucketFileNameMapping.entrySet().iterator();
         aliasBucketBaseFileNameMapping = new LinkedHashMap<String, LinkedHashMap<String, ArrayList<String>>>();
-        
+
         while (iter.hasNext()) {
           Entry<String, LinkedHashMap<String, ArrayList<String>>> old = iter.next();
-          
+
           LinkedHashMap<String, ArrayList<String>> newBucketBaseFileNameMapping = new LinkedHashMap<String, ArrayList<String>>();
           Iterator<Entry<String, ArrayList<String>>> oldAliasFileNameMappingIter = old.getValue().entrySet().iterator();
           while (oldAliasFileNameMappingIter.hasNext()) {
@@ -139,7 +146,7 @@ public class MapredLocalWork implements 
             if (oldTableBucketNames != null) {
               for (String bucketFName : oldTableBucketNames) {
                 newTableBucketFileBaseName.add(getBaseFileName(bucketFName));
-              }              
+              }
             }
             String bigTblBucketFileName = getBaseFileName(oldTableBucketFileNames.getKey());
             if(newBucketBaseFileNameMapping.containsKey(bigTblBucketFileName)) {
@@ -155,7 +162,7 @@ public class MapredLocalWork implements 
         }
       }
     }
-    
+
     private String getBaseFileName (String path) {
       try {
         URI uri = new URI(path);
@@ -185,12 +192,12 @@ public class MapredLocalWork implements 
     public LinkedHashMap<String, LinkedHashMap<String, ArrayList<String>>> getAliasBucketFileNameMapping() {
       return aliasBucketFileNameMapping;
     }
-    
+
     public void setAliasBucketFileNameMapping(
         LinkedHashMap<String, LinkedHashMap<String, ArrayList<String>>> aliasBucketFileNameMapping) {
       this.aliasBucketFileNameMapping = aliasBucketFileNameMapping;
     }
-    
+
     public String toString() {
       if (aliasBucketFileNameMapping != null)
         return "Mapping:" + aliasBucketFileNameMapping.toString();

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java?rev=922404&r1=922403&r2=922404&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java Fri Mar 12 19:53:57 2010
@@ -25,9 +25,9 @@ import java.util.Enumeration;
 import java.util.Properties;
 
 import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils;
 import org.apache.hadoop.hive.ql.io.HiveOutputFormat;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.serde2.Deserializer;
 import org.apache.hadoop.mapred.InputFormat;
 
@@ -243,6 +243,8 @@ public class PartitionDesc implements Se
    *          URI to the partition file
    */
   void deriveBaseFileName(String path) {
+    PlanUtils.configureTableJobPropertiesForStorageHandler(tableDesc);
+    
     if (path == null) {
       return;
     }

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java?rev=922404&r1=922403&r2=922404&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java Fri Mar 12 19:53:57 2010
@@ -24,6 +24,7 @@ import java.util.Collections;
 import java.util.Comparator;
 import java.util.LinkedHashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Properties;
 
 import org.apache.commons.logging.Log;
@@ -36,6 +37,10 @@ import org.apache.hadoop.hive.ql.exec.Ro
 import org.apache.hadoop.hive.ql.exec.Utilities;
 import org.apache.hadoop.hive.ql.io.HiveOutputFormat;
 import org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat;
+import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.HiveUtils;
+import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler;
 import org.apache.hadoop.hive.ql.parse.SemanticException;
 import org.apache.hadoop.hive.ql.parse.TypeCheckProcFactory;
 import org.apache.hadoop.hive.serde.Constants;
@@ -481,6 +486,42 @@ public final class PlanUtils {
         tag, partitionCols, order.toString(), numReducers);
   }
 
+  /**
+   * Loads the storage handler (if one exists) for the given table
+   * and invokes {@link HiveStorageHandler#configureTableJobProperties}.
+   *
+   * @param tableDesc table descriptor
+   */
+  public static void configureTableJobPropertiesForStorageHandler(
+    TableDesc tableDesc) {
+
+    if (tableDesc == null) {
+      return;
+    }
+
+    try {
+      HiveStorageHandler storageHandler =
+        HiveUtils.getStorageHandler(
+          Hive.get().getConf(),
+          tableDesc.getProperties().getProperty(
+            org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_STORAGE));
+      if (storageHandler != null) {
+        Map<String, String> jobProperties = new LinkedHashMap<String, String>();
+        storageHandler.configureTableJobProperties(
+          tableDesc,
+          jobProperties);
+        // Job properties are only relevant for non-native tables, so
+        // for native tables, leave it null to avoid cluttering up
+        // plans.
+        if (!jobProperties.isEmpty()) {
+          tableDesc.setJobProperties(jobProperties);
+        }
+      }
+    } catch (HiveException ex) {
+      throw new RuntimeException(ex);
+    }
+  }
+
   private PlanUtils() {
     // prevent instantiation
   }

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/TableDesc.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/TableDesc.java?rev=922404&r1=922403&r2=922404&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/TableDesc.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/TableDesc.java Fri Mar 12 19:53:57 2010
@@ -21,6 +21,8 @@ package org.apache.hadoop.hive.ql.plan;
 import java.io.Serializable;
 import java.util.Enumeration;
 import java.util.Properties;
+import java.util.LinkedHashMap;
+import java.util.Map;
 
 import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils;
 import org.apache.hadoop.hive.ql.io.HiveOutputFormat;
@@ -38,6 +40,7 @@ public class TableDesc implements Serial
   private Class<? extends HiveOutputFormat> outputFileFormatClass;
   private java.util.Properties properties;
   private String serdeClassName;
+  private Map<String, String> jobProperties;
 
   public TableDesc() {
   }
@@ -100,6 +103,15 @@ public class TableDesc implements Serial
     this.properties = properties;
   }
 
+  public void setJobProperties(Map<String, String> jobProperties) {
+    this.jobProperties = jobProperties;
+  }
+
+  @Explain(displayName = "jobProperties", normalExplain = false)
+  public Map<String, String> getJobProperties() {
+    return jobProperties;
+  }
+
   /**
    * @return the serdeClassName
    */
@@ -132,6 +144,12 @@ public class TableDesc implements Serial
     return getOutputFileFormatClass().getName();
   }
 
+  public boolean isNonNative() {
+    return (properties.getProperty(
+        org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_STORAGE)
+      != null);
+  }
+  
   @Override
   public Object clone() {
     TableDesc ret = new TableDesc();
@@ -147,6 +165,9 @@ public class TableDesc implements Serial
     }
 
     ret.setProperties(newProp);
+    if (jobProperties != null) {
+      ret.jobProperties = new LinkedHashMap<String, String>(jobProperties);
+    }
     return ret;
   }
 }

Modified: hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java?rev=922404&r1=922403&r2=922404&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java (original)
+++ hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java Fri Mar 12 19:53:57 2010
@@ -84,7 +84,7 @@ public class QTestUtil {
 
   private ParseDriver pd;
   private Hive db;
-  private final HiveConf conf;
+  protected final HiveConf conf;
   private Driver drv;
   private SemanticAnalyzer sem;
   private FileSystem fs;
@@ -199,9 +199,16 @@ public class QTestUtil {
     }
 
     srcTables = new LinkedList<String>();
+
+    preTestUtilInit();
     init();
   }
 
+  protected void preTestUtilInit() throws Exception {
+    // do some initialization before we setup the hive
+    // for example, in a HBase Test Util, we setup a hbase mini cluster here
+  }
+
   public void shutdown() throws Exception {
     cleanUp();
 

Added: hadoop/hive/trunk/ql/src/test/queries/clientnegative/alter_non_native.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientnegative/alter_non_native.q?rev=922404&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientnegative/alter_non_native.q (added)
+++ hadoop/hive/trunk/ql/src/test/queries/clientnegative/alter_non_native.q Fri Mar 12 19:53:57 2010
@@ -0,0 +1,6 @@
+DROP TABLE non_native1;
+CREATE TABLE non_native1(key int, value string) 
+STORED BY 'org.apache.hadoop.hive.ql.metadata.DefaultStorageHandler';
+
+-- we do not support ALTER TABLE on non-native tables yet
+ALTER TABLE non_native1 RENAME TO new_non_native;

Added: hadoop/hive/trunk/ql/src/test/queries/clientnegative/load_non_native.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientnegative/load_non_native.q?rev=922404&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientnegative/load_non_native.q (added)
+++ hadoop/hive/trunk/ql/src/test/queries/clientnegative/load_non_native.q Fri Mar 12 19:53:57 2010
@@ -0,0 +1,5 @@
+DROP TABLE non_native2;
+CREATE TABLE non_native2(key int, value string) 
+STORED BY 'org.apache.hadoop.hive.ql.metadata.DefaultStorageHandler';
+
+LOAD DATA LOCAL INPATH '../data/files/kv1.txt' INTO TABLE non_native2;

Added: hadoop/hive/trunk/ql/src/test/results/clientnegative/alter_non_native.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientnegative/alter_non_native.q.out?rev=922404&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientnegative/alter_non_native.q.out (added)
+++ hadoop/hive/trunk/ql/src/test/results/clientnegative/alter_non_native.q.out Fri Mar 12 19:53:57 2010
@@ -0,0 +1,16 @@
+PREHOOK: query: DROP TABLE non_native1
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE non_native1
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE non_native1(key int, value string) 
+STORED BY 'org.apache.hadoop.hive.ql.metadata.DefaultStorageHandler'
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE non_native1(key int, value string) 
+STORED BY 'org.apache.hadoop.hive.ql.metadata.DefaultStorageHandler'
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@non_native1
+PREHOOK: query: -- we do not support ALTER TABLE on non-native tables yet
+ALTER TABLE non_native1 RENAME TO new_non_native
+PREHOOK: type: ALTERTABLE_RENAME
+FAILED: Error in metadata: Cannot use ALTER TABLE on a non-native table
+FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask

Added: hadoop/hive/trunk/ql/src/test/results/clientnegative/load_non_native.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientnegative/load_non_native.q.out?rev=922404&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientnegative/load_non_native.q.out (added)
+++ hadoop/hive/trunk/ql/src/test/results/clientnegative/load_non_native.q.out Fri Mar 12 19:53:57 2010
@@ -0,0 +1,12 @@
+PREHOOK: query: DROP TABLE non_native2
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE non_native2
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE non_native2(key int, value string) 
+STORED BY 'org.apache.hadoop.hive.ql.metadata.DefaultStorageHandler'
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE non_native2(key int, value string) 
+STORED BY 'org.apache.hadoop.hive.ql.metadata.DefaultStorageHandler'
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@non_native2
+FAILED: Error in semantic analysis: A non-native table cannot be used as target for LOAD

Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyMap.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyMap.java?rev=922404&r1=922403&r2=922404&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyMap.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyMap.java Fri Mar 12 19:53:57 2010
@@ -288,7 +288,7 @@ public class LazyMap extends LazyNonPrim
    * cachedMap is reused for different calls to getMap(). But each LazyMap has a
    * separate cachedMap so we won't overwrite the data by accident.
    */
-  LinkedHashMap<Object, Object> cachedMap;
+  protected LinkedHashMap<Object, Object> cachedMap;
 
   /**
    * Return the map object representing this LazyMap. Note that the keyObjects
@@ -336,4 +336,11 @@ public class LazyMap extends LazyNonPrim
     return mapSize;
   }
 
+  protected boolean getParsed() {
+    return parsed;
+  }
+
+  protected void setParsed(boolean parsed) {
+    this.parsed = parsed;
+  }
 }

Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyObject.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyObject.java?rev=922404&r1=922403&r2=922404&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyObject.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyObject.java Fri Mar 12 19:53:57 2010
@@ -63,4 +63,12 @@ public abstract class LazyObject<OI exte
 
   @Override
   public abstract int hashCode();
+
+  protected OI getInspector() {
+    return oi;
+  }
+
+  protected void setInspector(OI oi) {
+    this.oi = oi;
+  }
 }

Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyStruct.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyStruct.java?rev=922404&r1=922403&r2=922404&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyStruct.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyStruct.java Fri Mar 12 19:53:57 2010
@@ -241,4 +241,27 @@ public class LazyStruct extends
     return this;
   }
 
+  protected boolean getParsed() {
+    return parsed;
+  }
+  
+  protected void setParsed(boolean parsed) {
+    this.parsed = parsed;
+  }
+
+  protected LazyObject[] getFields() {
+    return fields;
+  }
+
+  protected void setFields(LazyObject[] fields) {
+    this.fields = fields;
+  }
+  
+  protected boolean[] getFieldInited() {
+    return fieldInited;
+  }
+  
+  protected void setFieldInited(boolean[] fieldInited) {
+    this.fieldInited = fieldInited;
+  }
 }



Mime
View raw message