hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From prasan...@apache.org
Subject hive git commit: HIVE-11658: Load data file format validation does not work with directories (Prasanth Jayachandran reviewed by Gunther Hagleitner)
Date Thu, 27 Aug 2015 16:46:54 GMT
Repository: hive
Updated Branches:
  refs/heads/branch-1 23878debd -> 653507654


HIVE-11658: Load data file format validation does not work with directories (Prasanth Jayachandran
reviewed by Gunther Hagleitner)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/65350765
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/65350765
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/65350765

Branch: refs/heads/branch-1
Commit: 653507654f0e4ac7a31e65a59eabf62415aaf770
Parents: 23878de
Author: Prasanth Jayachandran <j.prasanth.j@gmail.com>
Authored: Thu Aug 27 11:46:40 2015 -0500
Committer: Prasanth Jayachandran <j.prasanth.j@gmail.com>
Committed: Thu Aug 27 11:46:40 2015 -0500

----------------------------------------------------------------------
 .../hive/ql/parse/LoadSemanticAnalyzer.java     |  38 +-
 .../ql/parse/LoadSemanticAnalyzer.java.orig     | 366 +++++++++++++++++++
 .../queries/clientnegative/load_orc_negative3.q |   6 +
 .../test/queries/clientpositive/load_orc_part.q |   4 +
 .../clientnegative/load_orc_negative3.q.out     |  25 ++
 .../results/clientpositive/load_orc_part.q.out  |  18 +
 6 files changed, 441 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/65350765/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
index 944cee4..0051dae 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
@@ -128,9 +128,11 @@ public class LoadSemanticAnalyzer extends BaseSemanticAnalyzer {
     return new URI(fromScheme, fromAuthority, path, null, null);
   }
 
-  private void applyConstraints(URI fromURI, URI toURI, Tree ast,
+  private FileStatus[] applyConstraintsAndGetFiles(URI fromURI, URI toURI, Tree ast,
       boolean isLocal) throws SemanticException {
 
+    FileStatus[] srcs = null;
+
     // local mode implies that scheme should be "file"
     // we can change this going forward
     if (isLocal && !fromURI.getScheme().equals("file")) {
@@ -139,7 +141,7 @@ public class LoadSemanticAnalyzer extends BaseSemanticAnalyzer {
     }
 
     try {
-      FileStatus[] srcs = matchFilesOrDir(FileSystem.get(fromURI, conf), new Path(fromURI));
+      srcs = matchFilesOrDir(FileSystem.get(fromURI, conf), new Path(fromURI));
       if (srcs == null || srcs.length == 0) {
         throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg(ast,
             "No files matching path " + fromURI));
@@ -168,6 +170,8 @@ public class LoadSemanticAnalyzer extends BaseSemanticAnalyzer {
           + "\"hive.metastore.warehouse.dir\" do not conflict.";
       throw new SemanticException(ErrorMsg.ILLEGAL_PATH.getMsg(ast, reason));
     }
+
+    return srcs;
   }
 
   @Override
@@ -232,11 +236,11 @@ public class LoadSemanticAnalyzer extends BaseSemanticAnalyzer {
     }
 
     // make sure the arguments make sense
-    applyConstraints(fromURI, toURI, fromTree, isLocal);
+    FileStatus[] files = applyConstraintsAndGetFiles(fromURI, toURI, fromTree, isLocal);
 
     // for managed tables, make sure the file formats match
     if (TableType.MANAGED_TABLE.equals(ts.tableHandle.getTableType())) {
-      ensureFileFormatsMatch(ts, fromURI);
+      ensureFileFormatsMatch(ts, files);
     }
     inputs.add(toReadEntity(new Path(fromURI)));
     Task<? extends Serializable> rTask = null;
@@ -334,7 +338,7 @@ public class LoadSemanticAnalyzer extends BaseSemanticAnalyzer {
     }
   }
 
-  private void ensureFileFormatsMatch(TableSpec ts, URI fromURI) throws SemanticException
{
+  private void ensureFileFormatsMatch(TableSpec ts, FileStatus[] fileStatuses) throws SemanticException
{
     final Class<? extends InputFormat> destInputFormat;
     try {
       if (ts.getPartSpec() == null || ts.getPartSpec().isEmpty()) {
@@ -349,17 +353,19 @@ public class LoadSemanticAnalyzer extends BaseSemanticAnalyzer {
     // Other file formats should do similar check to make sure file formats match
     // when doing LOAD DATA .. INTO TABLE
     if (OrcInputFormat.class.equals(destInputFormat)) {
-      Path inputFilePath = new Path(fromURI);
-      try {
-        FileSystem fs = FileSystem.get(fromURI, conf);
-        // just creating orc reader is going to do sanity checks to make sure its valid ORC
file
-        OrcFile.createReader(fs, inputFilePath);
-      } catch (FileFormatException e) {
-        throw new SemanticException(ErrorMsg.INVALID_FILE_FORMAT_IN_LOAD.getMsg("Destination"
+
-            " table is stored as ORC but the file being loaded is not a valid ORC file."));
-      } catch (IOException e) {
-        throw new SemanticException("Unable to load data to destination table." +
-            " Error: " + e.getMessage());
+      for (FileStatus fileStatus : fileStatuses) {
+        try {
+          Path filePath = fileStatus.getPath();
+          FileSystem fs = FileSystem.get(filePath.toUri(), conf);
+          // just creating orc reader is going to do sanity checks to make sure its valid
ORC file
+          OrcFile.createReader(fs, filePath);
+        } catch (FileFormatException e) {
+          throw new SemanticException(ErrorMsg.INVALID_FILE_FORMAT_IN_LOAD.getMsg("Destination"
+
+              " table is stored as ORC but the file being loaded is not a valid ORC file."));
+        } catch (IOException e) {
+          throw new SemanticException("Unable to load data to destination table." +
+              " Error: " + e.getMessage());
+        }
       }
     }
   }

http://git-wip-us.apache.org/repos/asf/hive/blob/65350765/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java.orig
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java.orig b/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java.orig
new file mode 100644
index 0000000..944cee4
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java.orig
@@ -0,0 +1,366 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.parse;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.antlr.runtime.tree.Tree;
+import org.apache.commons.httpclient.util.URIUtil;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.TableType;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.ql.ErrorMsg;
+import org.apache.hadoop.hive.ql.exec.Task;
+import org.apache.hadoop.hive.ql.exec.TaskFactory;
+import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.hooks.WriteEntity;
+import org.apache.hadoop.hive.ql.io.FileFormatException;
+import org.apache.hadoop.hive.ql.io.orc.OrcFile;
+import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat;
+import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.Partition;
+import org.apache.hadoop.hive.ql.plan.LoadTableDesc;
+import org.apache.hadoop.hive.ql.plan.MoveWork;
+import org.apache.hadoop.hive.ql.plan.StatsWork;
+import org.apache.hadoop.mapred.InputFormat;
+
+/**
+ * LoadSemanticAnalyzer.
+ *
+ */
+public class LoadSemanticAnalyzer extends BaseSemanticAnalyzer {
+
+  public LoadSemanticAnalyzer(HiveConf conf) throws SemanticException {
+    super(conf);
+  }
+
+  public static FileStatus[] matchFilesOrDir(FileSystem fs, Path path)
+      throws IOException {
+    FileStatus[] srcs = fs.globStatus(path, new PathFilter() {
+      @Override
+      public boolean accept(Path p) {
+        String name = p.getName();
+        return name.equals("_metadata") ? true : !name.startsWith("_") && !name.startsWith(".");
+      }
+    });
+    if ((srcs != null) && srcs.length == 1) {
+      if (srcs[0].isDir()) {
+        srcs = fs.listStatus(srcs[0].getPath(), new PathFilter() {
+          @Override
+          public boolean accept(Path p) {
+            String name = p.getName();
+            return !name.startsWith("_") && !name.startsWith(".");
+          }
+        });
+      }
+    }
+    return (srcs);
+  }
+
+  private URI initializeFromURI(String fromPath, boolean isLocal) throws IOException,
+      URISyntaxException {
+    URI fromURI = new Path(fromPath).toUri();
+
+    String fromScheme = fromURI.getScheme();
+    String fromAuthority = fromURI.getAuthority();
+    String path = fromURI.getPath();
+
+    // generate absolute path relative to current directory or hdfs home
+    // directory
+    if (!path.startsWith("/")) {
+      if (isLocal) {
+        path = URIUtil.decode(
+            new Path(System.getProperty("user.dir"), fromPath).toUri().toString());
+      } else {
+        path = new Path(new Path("/user/" + System.getProperty("user.name")),
+          path).toString();
+      }
+    }
+
+    // set correct scheme and authority
+    if (StringUtils.isEmpty(fromScheme)) {
+      if (isLocal) {
+        // file for local
+        fromScheme = "file";
+      } else {
+        // use default values from fs.default.name
+        URI defaultURI = FileSystem.get(conf).getUri();
+        fromScheme = defaultURI.getScheme();
+        fromAuthority = defaultURI.getAuthority();
+      }
+    }
+
+    // if scheme is specified but not authority then use the default authority
+    if ((!fromScheme.equals("file")) && StringUtils.isEmpty(fromAuthority)) {
+      URI defaultURI = FileSystem.get(conf).getUri();
+      fromAuthority = defaultURI.getAuthority();
+    }
+
+    LOG.debug(fromScheme + "@" + fromAuthority + "@" + path);
+    return new URI(fromScheme, fromAuthority, path, null, null);
+  }
+
+  private void applyConstraints(URI fromURI, URI toURI, Tree ast,
+      boolean isLocal) throws SemanticException {
+
+    // local mode implies that scheme should be "file"
+    // we can change this going forward
+    if (isLocal && !fromURI.getScheme().equals("file")) {
+      throw new SemanticException(ErrorMsg.ILLEGAL_PATH.getMsg(ast,
+          "Source file system should be \"file\" if \"local\" is specified"));
+    }
+
+    try {
+      FileStatus[] srcs = matchFilesOrDir(FileSystem.get(fromURI, conf), new Path(fromURI));
+      if (srcs == null || srcs.length == 0) {
+        throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg(ast,
+            "No files matching path " + fromURI));
+      }
+
+      for (FileStatus oneSrc : srcs) {
+        if (oneSrc.isDir()) {
+          throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg(ast,
+              "source contains directory: " + oneSrc.getPath().toString()));
+        }
+      }
+    } catch (IOException e) {
+      // Has to use full name to make sure it does not conflict with
+      // org.apache.commons.lang.StringUtils
+      throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg(ast), e);
+    }
+
+    // only in 'local' mode do we copy stuff from one place to another.
+    // reject different scheme/authority in other cases.
+    if (!isLocal
+        && (!StringUtils.equals(fromURI.getScheme(), toURI.getScheme()) || !StringUtils
+        .equals(fromURI.getAuthority(), toURI.getAuthority()))) {
+      String reason = "Move from: " + fromURI.toString() + " to: "
+          + toURI.toString() + " is not valid. "
+          + "Please check that values for params \"default.fs.name\" and "
+          + "\"hive.metastore.warehouse.dir\" do not conflict.";
+      throw new SemanticException(ErrorMsg.ILLEGAL_PATH.getMsg(ast, reason));
+    }
+  }
+
+  @Override
+  public void analyzeInternal(ASTNode ast) throws SemanticException {
+    boolean isLocal = false;
+    boolean isOverWrite = false;
+    Tree fromTree = ast.getChild(0);
+    Tree tableTree = ast.getChild(1);
+
+    if (ast.getChildCount() == 4) {
+      isLocal = true;
+      isOverWrite = true;
+    }
+
+    if (ast.getChildCount() == 3) {
+      if (ast.getChild(2).getText().toLowerCase().equals("local")) {
+        isLocal = true;
+      } else {
+        isOverWrite = true;
+      }
+    }
+
+    // initialize load path
+    URI fromURI;
+    try {
+      String fromPath = stripQuotes(fromTree.getText());
+      fromURI = initializeFromURI(fromPath, isLocal);
+    } catch (IOException e) {
+      throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg(fromTree, e
+          .getMessage()), e);
+    } catch (URISyntaxException e) {
+      throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg(fromTree, e
+          .getMessage()), e);
+    }
+
+    // initialize destination table/partition
+    TableSpec ts = new TableSpec(db, conf, (ASTNode) tableTree);
+
+    if (ts.tableHandle.isOffline()){
+      throw new SemanticException(
+          ErrorMsg.OFFLINE_TABLE_OR_PARTITION.getMsg(":Table " + ts.tableName));
+    }
+
+    if (ts.tableHandle.isView()) {
+      throw new SemanticException(ErrorMsg.DML_AGAINST_VIEW.getMsg());
+    }
+    if (ts.tableHandle.isNonNative()) {
+      throw new SemanticException(ErrorMsg.LOAD_INTO_NON_NATIVE.getMsg());
+    }
+
+    if(ts.tableHandle.isStoredAsSubDirectories()) {
+      throw new SemanticException(ErrorMsg.LOAD_INTO_STORED_AS_DIR.getMsg());
+    }
+
+    URI toURI = ((ts.partHandle != null) ? ts.partHandle.getDataLocation()
+        : ts.tableHandle.getDataLocation()).toUri();
+
+    List<FieldSchema> parts = ts.tableHandle.getPartitionKeys();
+    if ((parts != null && parts.size() > 0)
+        && (ts.partSpec == null || ts.partSpec.size() == 0)) {
+      throw new SemanticException(ErrorMsg.NEED_PARTITION_ERROR.getMsg());
+    }
+
+    // make sure the arguments make sense
+    applyConstraints(fromURI, toURI, fromTree, isLocal);
+
+    // for managed tables, make sure the file formats match
+    if (TableType.MANAGED_TABLE.equals(ts.tableHandle.getTableType())) {
+      ensureFileFormatsMatch(ts, fromURI);
+    }
+    inputs.add(toReadEntity(new Path(fromURI)));
+    Task<? extends Serializable> rTask = null;
+
+    // create final load/move work
+
+    boolean preservePartitionSpecs = false;
+
+    Map<String, String> partSpec = ts.getPartSpec();
+    if (partSpec == null) {
+      partSpec = new LinkedHashMap<String, String>();
+      outputs.add(new WriteEntity(ts.tableHandle,
+          (isOverWrite ? WriteEntity.WriteType.INSERT_OVERWRITE :
+              WriteEntity.WriteType.INSERT)));
+    } else {
+      try{
+        Partition part = Hive.get().getPartition(ts.tableHandle, partSpec, false);
+        if (part != null) {
+          if (part.isOffline()) {
+            throw new SemanticException(ErrorMsg.OFFLINE_TABLE_OR_PARTITION.
+                getMsg(ts.tableName + ":" + part.getName()));
+          }
+          if (isOverWrite){
+            outputs.add(new WriteEntity(part, WriteEntity.WriteType.INSERT_OVERWRITE));
+          } else {
+            outputs.add(new WriteEntity(part, WriteEntity.WriteType.INSERT));
+            // If partition already exists and we aren't overwriting it, then respect
+            // its current location info rather than picking it from the parent TableDesc
+            preservePartitionSpecs = true;
+          }
+        } else {
+          outputs.add(new WriteEntity(ts.tableHandle,
+          (isOverWrite ? WriteEntity.WriteType.INSERT_OVERWRITE :
+              WriteEntity.WriteType.INSERT)));
+        }
+      } catch(HiveException e) {
+        throw new SemanticException(e);
+      }
+    }
+
+
+    LoadTableDesc loadTableWork;
+    loadTableWork = new LoadTableDesc(new Path(fromURI),
+      Utilities.getTableDesc(ts.tableHandle), partSpec, isOverWrite);
+    if (preservePartitionSpecs){
+      // Note : preservePartitionSpecs=true implies inheritTableSpecs=false but
+      // but preservePartitionSpecs=false(default) here is not sufficient enough
+      // info to set inheritTableSpecs=true
+      loadTableWork.setInheritTableSpecs(false);
+    }
+
+    Task<? extends Serializable> childTask = TaskFactory.get(new MoveWork(getInputs(),
+        getOutputs(), loadTableWork, null, true, isLocal), conf);
+    if (rTask != null) {
+      rTask.addDependentTask(childTask);
+    } else {
+      rTask = childTask;
+    }
+
+    rootTasks.add(rTask);
+
+    // The user asked for stats to be collected.
+    // Some stats like number of rows require a scan of the data
+    // However, some other stats, like number of files, do not require a complete scan
+    // Update the stats which do not require a complete scan.
+    Task<? extends Serializable> statTask = null;
+    if (conf.getBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER)) {
+      StatsWork statDesc = new StatsWork(loadTableWork);
+      statDesc.setNoStatsAggregator(true);
+      statDesc.setClearAggregatorStats(true);
+      statDesc.setStatsReliable(conf.getBoolVar(HiveConf.ConfVars.HIVE_STATS_RELIABLE));
+      statTask = TaskFactory.get(statDesc, conf);
+    }
+
+    // HIVE-3334 has been filed for load file with index auto update
+    if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVEINDEXAUTOUPDATE)) {
+      IndexUpdater indexUpdater = new IndexUpdater(loadTableWork, getInputs(), conf);
+      try {
+        List<Task<? extends Serializable>> indexUpdateTasks = indexUpdater.generateUpdateTasks();
+
+        for (Task<? extends Serializable> updateTask : indexUpdateTasks) {
+          //LOAD DATA will either have a copy & move or just a move,
+          // we always want the update to be dependent on the move
+          childTask.addDependentTask(updateTask);
+          if (statTask != null) {
+            updateTask.addDependentTask(statTask);
+          }
+        }
+      } catch (HiveException e) {
+        console.printInfo("WARNING: could not auto-update stale indexes, indexes are not
out of sync");
+      }
+    }
+    else if (statTask != null) {
+      childTask.addDependentTask(statTask);
+    }
+  }
+
+  private void ensureFileFormatsMatch(TableSpec ts, URI fromURI) throws SemanticException
{
+    final Class<? extends InputFormat> destInputFormat;
+    try {
+      if (ts.getPartSpec() == null || ts.getPartSpec().isEmpty()) {
+        destInputFormat = ts.tableHandle.getInputFormatClass();
+      } else {
+        destInputFormat = ts.partHandle.getInputFormatClass();
+      }
+    } catch (HiveException e) {
+      throw new SemanticException(e);
+    }
+
+    // Other file formats should do similar check to make sure file formats match
+    // when doing LOAD DATA .. INTO TABLE
+    if (OrcInputFormat.class.equals(destInputFormat)) {
+      Path inputFilePath = new Path(fromURI);
+      try {
+        FileSystem fs = FileSystem.get(fromURI, conf);
+        // just creating orc reader is going to do sanity checks to make sure its valid ORC
file
+        OrcFile.createReader(fs, inputFilePath);
+      } catch (FileFormatException e) {
+        throw new SemanticException(ErrorMsg.INVALID_FILE_FORMAT_IN_LOAD.getMsg("Destination"
+
+            " table is stored as ORC but the file being loaded is not a valid ORC file."));
+      } catch (IOException e) {
+        throw new SemanticException("Unable to load data to destination table." +
+            " Error: " + e.getMessage());
+      }
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/65350765/ql/src/test/queries/clientnegative/load_orc_negative3.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientnegative/load_orc_negative3.q b/ql/src/test/queries/clientnegative/load_orc_negative3.q
new file mode 100644
index 0000000..9a4116e
--- /dev/null
+++ b/ql/src/test/queries/clientnegative/load_orc_negative3.q
@@ -0,0 +1,6 @@
+create table text_test (userid bigint, string1 string, subtype double, decimal1 decimal,
ts timestamp);
+load data local inpath '../../data/files/kv1.txt' into table text_test;
+
+set hive.default.fileformat=ORC;
+create table orc_test (userid bigint, string1 string, subtype double, decimal1 decimal, ts
timestamp);
+load data inpath '${hiveconf:hive.metastore.warehouse.dir}/text_test/' into table orc_test;

http://git-wip-us.apache.org/repos/asf/hive/blob/65350765/ql/src/test/queries/clientpositive/load_orc_part.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/load_orc_part.q b/ql/src/test/queries/clientpositive/load_orc_part.q
index 0927ea4..2ff884d 100644
--- a/ql/src/test/queries/clientpositive/load_orc_part.q
+++ b/ql/src/test/queries/clientpositive/load_orc_part.q
@@ -9,6 +9,10 @@ load data inpath '${hiveconf:hive.metastore.warehouse.dir}/orc_staging/orc_split
 load data local inpath '../../data/files/orc_split_elim.orc' into table orc_test partition
(ds='10');
 dfs -ls ${hiveconf:hive.metastore.warehouse.dir}/orc_test/ds=10/;
 
+load data local inpath '../../data/files/orc_split_elim.orc' overwrite into table orc_staging;
+load data inpath '${hiveconf:hive.metastore.warehouse.dir}/orc_staging/' overwrite into table
orc_test partition (ds='10');
+dfs -ls ${hiveconf:hive.metastore.warehouse.dir}/orc_test/ds=10/;
+
 alter table orc_test add partition(ds='11');
 alter table orc_test partition(ds='11') set fileformat textfile;
 load data local inpath '../../data/files/kv1.txt' into table orc_test partition(ds='11');

http://git-wip-us.apache.org/repos/asf/hive/blob/65350765/ql/src/test/results/clientnegative/load_orc_negative3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/load_orc_negative3.q.out b/ql/src/test/results/clientnegative/load_orc_negative3.q.out
new file mode 100644
index 0000000..77fb50e
--- /dev/null
+++ b/ql/src/test/results/clientnegative/load_orc_negative3.q.out
@@ -0,0 +1,25 @@
+PREHOOK: query: create table text_test (userid bigint, string1 string, subtype double, decimal1
decimal, ts timestamp)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@text_test
+POSTHOOK: query: create table text_test (userid bigint, string1 string, subtype double, decimal1
decimal, ts timestamp)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@text_test
+PREHOOK: query: load data local inpath '../../data/files/kv1.txt' into table text_test
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@text_test
+POSTHOOK: query: load data local inpath '../../data/files/kv1.txt' into table text_test
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@text_test
+PREHOOK: query: create table orc_test (userid bigint, string1 string, subtype double, decimal1
decimal, ts timestamp)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@orc_test
+POSTHOOK: query: create table orc_test (userid bigint, string1 string, subtype double, decimal1
decimal, ts timestamp)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@orc_test
+FAILED: SemanticException [Error 30019]: The file that you are trying to load does not match
the file format of the destination table. Destination table is stored as ORC but the file
being loaded is not a valid ORC file.

http://git-wip-us.apache.org/repos/asf/hive/blob/65350765/ql/src/test/results/clientpositive/load_orc_part.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/load_orc_part.q.out b/ql/src/test/results/clientpositive/load_orc_part.q.out
index 34ca493..2e02c2e 100644
--- a/ql/src/test/results/clientpositive/load_orc_part.q.out
+++ b/ql/src/test/results/clientpositive/load_orc_part.q.out
@@ -42,6 +42,24 @@ POSTHOOK: type: LOAD
 POSTHOOK: Output: default@orc_test@ds=10
 Found 2 items
 #### A masked pattern was here ####
+PREHOOK: query: load data local inpath '../../data/files/orc_split_elim.orc' overwrite into
table orc_staging
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@orc_staging
+POSTHOOK: query: load data local inpath '../../data/files/orc_split_elim.orc' overwrite into
table orc_staging
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@orc_staging
+#### A masked pattern was here ####
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@orc_test@ds=10
+#### A masked pattern was here ####
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@orc_test@ds=10
+Found 1 items
+#### A masked pattern was here ####
 PREHOOK: query: alter table orc_test add partition(ds='11')
 PREHOOK: type: ALTERTABLE_ADDPARTS
 PREHOOK: Output: default@orc_test


Mime
View raw message