hadoop-hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From na...@apache.org
Subject svn commit: r946325 - in /hadoop/hive/trunk: CHANGES.txt bin/ext/cli.sh bin/ext/rcfilecat.sh bin/ext/util/ bin/ext/util/execHiveCmd.sh bin/hive cli/src/java/org/apache/hadoop/hive/cli/RCFileCat.java
Date Wed, 19 May 2010 18:20:26 GMT
Author: namit
Date: Wed May 19 18:20:26 2010
New Revision: 946325

URL: http://svn.apache.org/viewvc?rev=946325&view=rev
Log:
HIVE-1351. rcfilecat for debugging
(Yongqiang He via namit)


Added:
    hadoop/hive/trunk/bin/ext/rcfilecat.sh
    hadoop/hive/trunk/bin/ext/util/
    hadoop/hive/trunk/bin/ext/util/execHiveCmd.sh
    hadoop/hive/trunk/cli/src/java/org/apache/hadoop/hive/cli/RCFileCat.java
Modified:
    hadoop/hive/trunk/CHANGES.txt
    hadoop/hive/trunk/bin/ext/cli.sh
    hadoop/hive/trunk/bin/hive

Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=946325&r1=946324&r2=946325&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Wed May 19 18:20:26 2010
@@ -81,6 +81,9 @@ Trunk -  Unreleased
     HIVE-801. Add support for row-wise IN
     (Paul Yang via namit)
 
+    HIVE-1351. rcfilecat for debugging
+    (Yongqiang He via namit)
+
   IMPROVEMENTS
     HIVE-983. Function from_unixtime takes long.
     (Ning Zhang via zshao)

Modified: hadoop/hive/trunk/bin/ext/cli.sh
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/bin/ext/cli.sh?rev=946325&r1=946324&r2=946325&view=diff
==============================================================================
--- hadoop/hive/trunk/bin/ext/cli.sh (original)
+++ hadoop/hive/trunk/bin/ext/cli.sh Wed May 19 18:20:26 2010
@@ -3,45 +3,7 @@ export SERVICE_LIST="${SERVICE_LIST}${TH
 
 cli () {
   CLASS=org.apache.hadoop.hive.cli.CliDriver
-
-  # cli specific code
-  if [ ! -f ${HIVE_LIB}/hive-cli-*.jar ]; then
-    echo "Missing Hive CLI Jar"
-    exit 3;
-  fi
-
-  if $cygwin; then
-    HIVE_LIB=`cygpath -w "$HIVE_LIB"`
-  fi
-
-  version=$($HADOOP version | awk '{if (NR == 1) {print $2;}}');
-
-  # Save the regex to a var to workaround quoting incompatabilities
-  # between Bash 3.1 and 3.2
-  version_re="^([[:digit:]]+)\.([[:digit:]]+)(\.([[:digit:]]+))?.*$"
-
-  if [[ "$version" =~ $version_re ]]; then
-      major_ver=${BASH_REMATCH[1]}
-      minor_ver=${BASH_REMATCH[2]}
-      patch_ver=${BASH_REMATCH[4]}
-  else
-      echo "Unable to determine Hadoop version information."
-      echo "'hadoop version' returned:"
-      echo `$HADOOP version`
-      exit 6
-  fi
-
-  # increase the threashold for large queries
-  HADOOP_HEAPSIZE=4096 
-  HADOOP_OPTS="$HADOOP_OPTS -XX:-UseGCOverheadLimit"
-
-  if [ $minor_ver -lt 20 ]; then
-      exec $HADOOP jar $AUX_JARS_CMD_LINE ${HIVE_LIB}/hive-cli-*.jar $CLASS $HIVE_OPTS "$@"
-  else
-      # hadoop 20 or newer - skip the aux_jars option. picked up from hiveconf
-      exec $HADOOP jar ${HIVE_LIB}/hive-cli-*.jar $CLASS $HIVE_OPTS "$@" 
-  fi
-
+  execHiveCmd $CLASS "$@"
 }
 
 cli_help () {

Added: hadoop/hive/trunk/bin/ext/rcfilecat.sh
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/bin/ext/rcfilecat.sh?rev=946325&view=auto
==============================================================================
--- hadoop/hive/trunk/bin/ext/rcfilecat.sh (added)
+++ hadoop/hive/trunk/bin/ext/rcfilecat.sh Wed May 19 18:20:26 2010
@@ -0,0 +1,11 @@
+THISSERVICE=rcfilecat
+export SERVICE_LIST="${SERVICE_LIST}${THISSERVICE} "
+
+rcfilecat () {
+  CLASS=org.apache.hadoop.hive.cli.RCFileCat
+  execHiveCmd $CLASS "$@"
+}
+
+rcfilecat_help () {
+  echo "usage ./hive rcfilecat [--start='startoffset'] [--length='len'] "
+} 

Added: hadoop/hive/trunk/bin/ext/util/execHiveCmd.sh
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/bin/ext/util/execHiveCmd.sh?rev=946325&view=auto
==============================================================================
--- hadoop/hive/trunk/bin/ext/util/execHiveCmd.sh (added)
+++ hadoop/hive/trunk/bin/ext/util/execHiveCmd.sh Wed May 19 18:20:26 2010
@@ -0,0 +1,43 @@
+execHiveCmd () {
+  CLASS=$1;
+  shift;
+
+  # cli specific code
+  if [ ! -f ${HIVE_LIB}/hive-cli-*.jar ]; then
+    echo "Missing Hive CLI Jar"
+    exit 3;
+  fi
+
+  if $cygwin; then
+    HIVE_LIB=`cygpath -w "$HIVE_LIB"`
+  fi
+
+  version=$($HADOOP version | awk '{if (NR == 1) {print $2;}}');
+
+  # Save the regex to a var to workaround quoting incompatabilities
+  # between Bash 3.1 and 3.2
+  version_re="^([[:digit:]]+)\.([[:digit:]]+)(\.([[:digit:]]+))?.*$"
+
+  if [[ "$version" =~ $version_re ]]; then
+      major_ver=${BASH_REMATCH[1]}
+      minor_ver=${BASH_REMATCH[2]}
+      patch_ver=${BASH_REMATCH[4]}
+  else
+      echo "Unable to determine Hadoop version information."
+      echo "'hadoop version' returned:"
+      echo `$HADOOP version`
+      exit 6
+  fi
+
+  # increase the threashold for large queries
+  HADOOP_HEAPSIZE=4096 
+  HADOOP_OPTS="$HADOOP_OPTS -XX:-UseGCOverheadLimit"
+
+  if [ $minor_ver -lt 20 ]; then
+      exec $HADOOP jar $AUX_JARS_CMD_LINE ${HIVE_LIB}/hive-cli-*.jar $CLASS $HIVE_OPTS "$@"
+  else
+      # hadoop 20 or newer - skip the aux_jars option. picked up from hiveconf
+      exec $HADOOP jar ${HIVE_LIB}/hive-cli-*.jar $CLASS $HIVE_OPTS "$@" 
+  fi
+
+}

Modified: hadoop/hive/trunk/bin/hive
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/bin/hive?rev=946325&r1=946324&r2=946325&view=diff
==============================================================================
--- hadoop/hive/trunk/bin/hive (original)
+++ hadoop/hive/trunk/bin/hive Wed May 19 18:20:26 2010
@@ -131,6 +131,10 @@ for i in "$bin"/ext/*.sh ; do
   . $i
 done
 
+for i in "$bin"/ext/util/*.sh ; do
+  . $i
+done
+
 SERVICE=""
 HELP=""
 while [ $# -gt 0 ]; do
@@ -140,6 +144,10 @@ while [ $# -gt 0 ]; do
       SERVICE=$1
       shift
       ;;
+    --rcfilecat)
+      SERVICE=rcfilecat
+      shift
+      ;;
     --help)
       HELP=_help
       shift

Added: hadoop/hive/trunk/cli/src/java/org/apache/hadoop/hive/cli/RCFileCat.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/cli/src/java/org/apache/hadoop/hive/cli/RCFileCat.java?rev=946325&view=auto
==============================================================================
--- hadoop/hive/trunk/cli/src/java/org/apache/hadoop/hive/cli/RCFileCat.java (added)
+++ hadoop/hive/trunk/cli/src/java/org/apache/hadoop/hive/cli/RCFileCat.java Wed May 19 18:20:26
2010
@@ -0,0 +1,121 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.cli;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.ql.io.RCFileRecordReader;
+import org.apache.hadoop.hive.serde2.columnar.BytesRefArrayWritable;
+import org.apache.hadoop.hive.serde2.columnar.BytesRefWritable;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.FileSplit;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+public class RCFileCat implements Tool{
+  
+  Configuration conf = null;
+
+  @Override
+  public int run(String[] args) throws Exception {
+    long start = 0l;
+    long length = -1l;
+    
+    //get options from arguments
+    if (args.length < 1 || args.length > 3) {
+      printUsage(null);
+    }
+    Path fileName = null;
+    for (int i = 0; i < args.length; i++) {
+      String arg = args[i];
+      if(arg.startsWith("--start=")) {
+        start = Long.parseLong(arg.substring("--start=".length()));
+      } else if (arg.startsWith("--length=")) {
+        length = Long.parseLong(arg.substring("--length=".length()));
+      } else if (fileName == null){
+        fileName = new Path(arg);
+      } else {
+        printUsage(null);
+      }
+    }
+ 
+    FileSystem fs = FileSystem.get(fileName.toUri(), conf);
+    long fileLen = fs.getFileStatus(fileName).getLen();
+    if (start < 0) {
+      start = 0;
+    }
+    if (start > fileLen) {
+      return 0;
+    }
+    if (length < 0 || (start + length) > fileLen) {
+      length = fileLen - start;
+    }
+    
+    //share the code with RecordReader.
+    FileSplit split = new FileSplit(fileName,start, length, new JobConf(conf));
+    RCFileRecordReader recordReader = new RCFileRecordReader(conf, split);
+    LongWritable key = new LongWritable();
+    BytesRefArrayWritable value = new BytesRefArrayWritable();
+    Text txt = new Text();
+    while (recordReader.next(key, value)) {
+      txt.clear();
+      for (int i = 0; i < value.size(); i++) {
+        BytesRefWritable v = value.get(i);
+        txt.set(v.getData(), v.getStart(), v.getLength());
+        System.out.print(txt.toString());
+      }
+    }
+    return 0;
+  }
+
+  @Override
+  public Configuration getConf() {
+    return conf;
+  }
+
+  @Override
+  public void setConf(Configuration conf) {
+    this.conf = conf;
+  }
+  
+  private static String Usage = "RCFileCat [--start=start_offet] [--length=len] fileName";
+  
+  public static void main(String[] args) {
+    try {
+      Configuration conf = new Configuration();
+      RCFileCat instance = new RCFileCat();
+      instance.setConf(conf);
+      ToolRunner.run(instance, args);
+    } catch (Exception e) {
+      printUsage(e.getMessage());
+    }
+  }
+  
+  private static void printUsage(String errorMsg) {
+    System.out.println(Usage);
+    if(errorMsg != null) {
+      System.out.println(errorMsg);      
+    }
+    System.exit(1);
+  }
+  
+}



Mime
View raw message