hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From na...@apache.org
Subject svn commit: r1056167 - in /hive/trunk: ./ common/src/java/org/apache/hadoop/hive/conf/ conf/ ql/src/java/org/apache/hadoop/hive/ql/index/compact/ ql/src/java/org/apache/hadoop/hive/ql/io/
Date Fri, 07 Jan 2011 02:28:31 GMT
Author: namit
Date: Fri Jan  7 02:28:30 2011
New Revision: 1056167

URL: http://svn.apache.org/viewvc?rev=1056167&view=rev
Log:
HIVE-1889 add an option (hive.index.compact.file.ignore.hdfs)
to ignore HDFS location stored in index files
(Yongqiang He via namit)


Modified:
    hive/trunk/CHANGES.txt
    hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
    hive/trunk/conf/hive-default.xml
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/compact/HiveCompactIndexInputFormat.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/compact/HiveCompactIndexResult.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java

Modified: hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hive/trunk/CHANGES.txt?rev=1056167&r1=1056166&r2=1056167&view=diff
==============================================================================
--- hive/trunk/CHANGES.txt (original)
+++ hive/trunk/CHANGES.txt Fri Jan  7 02:28:30 2011
@@ -672,6 +672,10 @@ Trunk -  Unreleased
     HIVE-1840 Support ALTER DATABASE to change database properties
     (Ning Zhang via namit)
 
+    HIVE-1889 add an option (hive.index.compact.file.ignore.hdfs)
+    to ignore HDFS location stored in index files
+    (Yongqiang He via namit)
+
   TESTS
 
     HIVE-1464. improve  test query performance

Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1056167&r1=1056166&r2=1056167&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Fri Jan  7 02:28:30
2011
@@ -333,8 +333,9 @@ public class HiveConf extends Configurat
     // Print column names in output
     HIVE_CLI_PRINT_HEADER("hive.cli.print.header", false),
 
-    HIVE_ERROR_ON_EMPTY_PARTITION("hive.error.on.empty.partition", false);
-
+    HIVE_ERROR_ON_EMPTY_PARTITION("hive.error.on.empty.partition", false),
+    
+    HIVE_INDEX_IGNORE_HDFS_LOC("hive.index.compact.file.ignore.hdfs", false),
     ;
 
 

Modified: hive/trunk/conf/hive-default.xml
URL: http://svn.apache.org/viewvc/hive/trunk/conf/hive-default.xml?rev=1056167&r1=1056166&r2=1056167&view=diff
==============================================================================
--- hive/trunk/conf/hive-default.xml (original)
+++ hive/trunk/conf/hive-default.xml Fri Jan  7 02:28:30 2011
@@ -847,5 +847,11 @@
   <description>Whether to throw an excpetion if dynamic partition insert generates
empty results.</description>
 </property>
 
+<property>
+  <name>hive.index.compact.file.ignore.hdfs</name>
+  <value>false</value>
+  <description>True the hdfs location stored in the index file will be igbored at runtime.

+  If the data got moved or the name of the cluster got changed, the index data should still
be usable.</description>
+</property>
 
 </configuration>

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/compact/HiveCompactIndexInputFormat.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/compact/HiveCompactIndexInputFormat.java?rev=1056167&r1=1056166&r2=1056167&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/compact/HiveCompactIndexInputFormat.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/compact/HiveCompactIndexInputFormat.java
Fri Jan  7 02:28:30 2011
@@ -44,7 +44,7 @@ public class HiveCompactIndexInputFormat
     for (Path dir : dirs) {
       PartitionDesc part = HiveFileFormatUtils
           .getPartitionDescFromPathRecursively(pathToPartitionInfo, dir,
-              IOPrepareCache.get().allocatePartitionDescMap());
+              IOPrepareCache.get().allocatePartitionDescMap(), true);
       // create a new InputFormat instance if this is the first time to see this
       // class
       Class inputFormatClass = part.getInputFileFormatClass();

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/compact/HiveCompactIndexResult.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/compact/HiveCompactIndexResult.java?rev=1056167&r1=1056166&r2=1056167&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/compact/HiveCompactIndexResult.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/compact/HiveCompactIndexResult.java
Fri Jan  7 02:28:30 2011
@@ -31,6 +31,7 @@ import org.apache.hadoop.fs.FSDataInputS
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.serde2.columnar.BytesRefWritable;
 import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe;
@@ -74,6 +75,7 @@ public class HiveCompactIndexResult {
 
   JobConf job = null;
   BytesRefWritable[] bytesRef = new BytesRefWritable[2];
+  boolean ignoreHdfsLoc = false;
 
   public HiveCompactIndexResult(String indexFile, JobConf conf) throws IOException,
       HiveException {
@@ -81,6 +83,7 @@ public class HiveCompactIndexResult {
 
     bytesRef[0] = new BytesRefWritable();
     bytesRef[1] = new BytesRefWritable();
+    ignoreHdfsLoc = HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_INDEX_IGNORE_HDFS_LOC);

 
     if (indexFile != null) {
       Path indexFilePath = new Path(indexFile);
@@ -128,6 +131,11 @@ public class HiveCompactIndexResult {
               + line.toString());
     }
     String bucketFileName = new String(bytes, 0, firstEnd);
+    
+    if (ignoreHdfsLoc) {
+      Path tmpPath = new Path(bucketFileName);
+      bucketFileName = tmpPath.toUri().getPath();
+    }
     IBucket bucket = buckets.get(bucketFileName);
     if (bucket == null) {
       bucket = new IBucket(bucketFileName);

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java?rev=1056167&r1=1056166&r2=1056167&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java Fri Jan 
7 02:28:30 2011
@@ -249,11 +249,19 @@ public final class HiveFileFormatUtils {
       Map<String, PartitionDesc> pathToPartitionInfo, Path dir,
       Map<Map<String, PartitionDesc>, Map<String, PartitionDesc>> cacheMap)
       throws IOException {
+    return getPartitionDescFromPathRecursively(pathToPartitionInfo, dir,
+        cacheMap, false);
+  }
+  
+  public static PartitionDesc getPartitionDescFromPathRecursively(
+      Map<String, PartitionDesc> pathToPartitionInfo, Path dir,
+      Map<Map<String, PartitionDesc>, Map<String, PartitionDesc>> cacheMap,
+      boolean ignoreSchema) throws IOException {
 
     PartitionDesc part = doGetPartitionDescFromPath(pathToPartitionInfo, dir);
     if (part == null
-        && (dir.toUri().getScheme() == null || dir.toUri().getScheme().trim()
-            .equals(""))) {
+        && (ignoreSchema || (dir.toUri().getScheme() == null || dir.toUri().getScheme().trim()
+            .equals("")))) {
 
       Map<String, PartitionDesc> newPathToPartitionInfo = null;
       if (cacheMap != null) {
@@ -275,7 +283,7 @@ public final class HiveFileFormatUtils {
       return part;
     } else {
       throw new IOException("cannot find dir = " + dir.toString()
-                          + " in partToPartitionInfo: " + pathToPartitionInfo.keySet());
+                          + " in pathToPartitionInfo: " + pathToPartitionInfo.keySet());
     }
   }
 



Mime
View raw message