incubator-bigtop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aba...@apache.org
Subject svn commit: r1148501 [16/47] - in /incubator/bigtop: branches/ tags/ trunk/ trunk/docs/ trunk/src/ trunk/src/pkg/ trunk/src/pkg/common/ trunk/src/pkg/common/flume/ trunk/src/pkg/common/hadoop/ trunk/src/pkg/common/hadoop/conf.pseudo/ trunk/src/pkg/comm...
Date Tue, 19 Jul 2011 19:45:54 GMT
Added: incubator/bigtop/trunk/test/src/smokes/hbase/pom.xml
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/test/src/smokes/hbase/pom.xml?rev=1148501&view=auto
==============================================================================
--- incubator/bigtop/trunk/test/src/smokes/hbase/pom.xml (added)
+++ incubator/bigtop/trunk/test/src/smokes/hbase/pom.xml Tue Jul 19 19:44:48 2011
@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <parent>
+      <groupId>com.cloudera.itest</groupId>
+      <artifactId>cdh-smokes</artifactId>
+      <version>1.0-cdh3u1-SNAPSHOT</version>
+  </parent>
+  <groupId>com.cloudera.itest</groupId>
+  <artifactId>hbasesmoke</artifactId>
+  <version>0.90.1-cdh3u1-SNAPSHOT</version>
+  <name>hbasesmoke</name>
+</project>

Added: incubator/bigtop/trunk/test/src/smokes/hbase/src/test/groovy/com/cloudera/itest/hbasesmoke/TestHbasePigSmoke.groovy
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/test/src/smokes/hbase/src/test/groovy/com/cloudera/itest/hbasesmoke/TestHbasePigSmoke.groovy?rev=1148501&view=auto
==============================================================================
--- incubator/bigtop/trunk/test/src/smokes/hbase/src/test/groovy/com/cloudera/itest/hbasesmoke/TestHbasePigSmoke.groovy (added)
+++ incubator/bigtop/trunk/test/src/smokes/hbase/src/test/groovy/com/cloudera/itest/hbasesmoke/TestHbasePigSmoke.groovy Tue Jul 19 19:44:48 2011
@@ -0,0 +1,119 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.cloudera.itest.hbasesmoke
+
+import com.cloudera.itest.shell.Shell
+import org.junit.AfterClass
+import org.junit.BeforeClass
+import org.junit.Test
+import static junit.framework.Assert.assertEquals
+import static org.junit.Assert.assertTrue
+
+class TestHbasePigSmoke {
+  private static final int ROW_CNT = 10;
+
+  private static String extra_jars = 
+    System.getProperty("com.cloudera.itest.hbasesmoke.TestHbasePigSmoke.extra_jars",
+                                                        "");
+  private static String register_clause = "";
+  private static String tmp = "TestHbasePigSmoke-${(new Date().getTime())}";
+  private static String TABLE="smoke-${tmp}";
+  private static String FAM1='family1';
+  private static String FAM2='family2';
+
+  private static Shell shHbase = new Shell('hbase shell');
+  private static Shell shPig = new Shell('pig');
+  private static Shell sh = new Shell('/bin/bash -s');
+
+  static {
+    extra_jars.split(':').each {
+      register_clause <<= "register ${it};"
+    }
+  }
+
+  @BeforeClass
+  static void setUp() {
+    shHbase.exec("create '$TABLE', '$FAM1', '$FAM2'",
+                 "describe '$TABLE'",
+                 "quit\n");
+    assertEquals("Creating of the ${TABLE} failed",
+                 0, shHbase.ret);
+  }
+
+  @AfterClass
+  static void tearDown() {
+    shHbase.exec("disable '$TABLE'",
+                 "drop '$TABLE'",
+                 "quit\n");
+
+    sh.exec("hadoop fs -rmr $TABLE");
+  }
+
+  @Test(timeout=300000L)
+  public void Pig2Hbase() {
+    def script = "\n";
+
+    (1..ROW_CNT).each { script <<= String.format('%020d %d %s\n', it, it, 'localhost') }
+
+    sh.exec("hadoop dfs -mkdir $TABLE",
+            "hadoop dfs -put <(cat << __EOT__${script}__EOT__) ${TABLE}/data");
+    assertEquals("Can't copy data to HDFS",
+                 0, sh.ret);
+
+    shPig.exec("""
+      ${register_clause}
+      data = LOAD '$TABLE' using PigStorage(' ') as (cnt1, cnt2, name);
+      store data into '$TABLE' using org.apache.pig.backend.hadoop.hbase.HBaseStorage(
+             '$FAM1:cnt1 $FAM1:cnt2 $FAM2:name','-caster HBaseBinaryConverter');
+      quit
+      """);
+    assertEquals("Failed loading data via PIG",
+                 0, shPig.ret);
+
+    shHbase.exec("scan '$TABLE'",
+                 "quit\n");
+    assertTrue("Scanning the table returned wrong # of rows",
+               (shHbase.out.get(shHbase.out.size() - 3) =~ "^$ROW_CNT row.s. in .* seconds").find());
+  }
+
+  @Test(timeout=300000L)
+  public void Hbase2Pig() {
+    def script = "\n";
+
+    (1..10).each {
+      script <<= String.format("put '$TABLE', '%020d', '$FAM1:f1', '%s'\n", it, 'localhost');
+      script <<= String.format("put '$TABLE', '%020d', '$FAM2:f2', '%s'\n", it, 'localhost');
+    }
+    script << "quit\n\n";
+
+    shHbase.exec(script);
+
+    shPig.exec("""
+      ${register_clause}
+      hbaseData = LOAD '$TABLE' USING org.apache.pig.backend.hadoop.hbase.HBaseStorage(
+            '$FAM1:f1 $FAM2:f2', '-loadKey') as (rowkey, count, name) ;
+      store hbaseData into '${TABLE}/pig' using PigStorage(',');
+      quit
+      """);
+
+    sh.exec("hadoop fs -cat $TABLE/pig/part* | wc -l");
+    assertEquals("Scanning the PIG output returned wrong # of rows",
+                 ROW_CNT, sh.out.get(0).toInteger());
+  }
+}

Added: incubator/bigtop/trunk/test/src/smokes/hive/README
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/test/src/smokes/hive/README?rev=1148501&view=auto
==============================================================================
--- incubator/bigtop/trunk/test/src/smokes/hive/README (added)
+++ incubator/bigtop/trunk/test/src/smokes/hive/README Tue Jul 19 19:44:48 2011
@@ -0,0 +1,4 @@
+This is a project to develop and build Hive smoke and system tests.
+
+To be able to run this a started project is also required such as 
+  http://github.sf.cloudera.com/cos/iTest/blob/master/metaTest/hive.pom

Added: incubator/bigtop/trunk/test/src/smokes/hive/pom.xml
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/test/src/smokes/hive/pom.xml?rev=1148501&view=auto
==============================================================================
--- incubator/bigtop/trunk/test/src/smokes/hive/pom.xml (added)
+++ incubator/bigtop/trunk/test/src/smokes/hive/pom.xml Tue Jul 19 19:44:48 2011
@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <parent>
+      <groupId>com.cloudera.itest</groupId>
+      <artifactId>cdh-smokes</artifactId>
+      <version>1.0-cdh3u1-SNAPSHOT</version>
+  </parent>
+  <groupId>com.cloudera.itest</groupId>
+  <artifactId>hivesmoke</artifactId>
+  <version>0.7.0-cdh3u1-SNAPSHOT</version>
+  <name>hivesmoke</name>
+</project>

Added: incubator/bigtop/trunk/test/src/smokes/hive/src/test/groovy/com/cloudera/itest/hivesmoke/HiveBulkScriptExecutor.groovy
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/test/src/smokes/hive/src/test/groovy/com/cloudera/itest/hivesmoke/HiveBulkScriptExecutor.groovy?rev=1148501&view=auto
==============================================================================
--- incubator/bigtop/trunk/test/src/smokes/hive/src/test/groovy/com/cloudera/itest/hivesmoke/HiveBulkScriptExecutor.groovy (added)
+++ incubator/bigtop/trunk/test/src/smokes/hive/src/test/groovy/com/cloudera/itest/hivesmoke/HiveBulkScriptExecutor.groovy Tue Jul 19 19:44:48 2011
@@ -0,0 +1,66 @@
+package com.cloudera.itest.hivesmoke
+
+import com.cloudera.itest.JarContent
+import com.cloudera.itest.shell.Shell
+import static junit.framework.Assert.assertEquals
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+public class HiveBulkScriptExecutor {
+  static Shell sh = new Shell("/bin/bash -s");
+
+  private File scripts;
+  private String location;
+
+  public HiveBulkScriptExecutor(String l) {
+    location = l;
+    scripts = new File(location);
+
+    if (!scripts.exists()) {
+      JarContent.unpackJarContainer(HiveBulkScriptExecutor.class, '.' , null);
+    }
+  }
+
+  public List<String> getScripts() {
+    List<String> res = [];
+
+    try {
+      scripts.eachDir { res.add(it.name); }
+    } catch (Throwable ex) {}
+    return res;
+  }
+
+  public void runScript(String test, String extraArgs) {
+    String l = "${location}/${test}";
+    sh.exec("""
+    F=cat
+    if [ -f ${l}/filter ]; then
+      chmod 777 ${l}/filter
+      F=${l}/filter
+    fi
+    hive ${extraArgs} -v -f ${l}/in > ${l}/actual && diff -u <(\$F < ${l}/actual) <(\$F < ${l}/out)
+    """);
+    assertEquals("Got unexpected output from test script ${test}",
+                  0, sh.ret);
+  }
+
+  public void runScript(String test) {
+    runScript(test, "");
+  }
+}

Added: incubator/bigtop/trunk/test/src/smokes/hive/src/test/groovy/com/cloudera/itest/hivesmoke/IntegrationTestHiveSmokeBulk.groovy
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/test/src/smokes/hive/src/test/groovy/com/cloudera/itest/hivesmoke/IntegrationTestHiveSmokeBulk.groovy?rev=1148501&view=auto
==============================================================================
--- incubator/bigtop/trunk/test/src/smokes/hive/src/test/groovy/com/cloudera/itest/hivesmoke/IntegrationTestHiveSmokeBulk.groovy (added)
+++ incubator/bigtop/trunk/test/src/smokes/hive/src/test/groovy/com/cloudera/itest/hivesmoke/IntegrationTestHiveSmokeBulk.groovy Tue Jul 19 19:44:48 2011
@@ -0,0 +1,83 @@
+package com.cloudera.itest.hivesmoke
+
+import org.junit.Test
+import org.junit.AfterClass
+import org.junit.BeforeClass
+import com.cloudera.itest.shell.Shell
+import static junit.framework.Assert.assertEquals
+import com.cloudera.itest.junit.OrderedParameterized
+import org.junit.runner.RunWith
+import org.junit.runners.Parameterized.Parameters
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+@RunWith(OrderedParameterized.class)
+public class IntegrationTestHiveSmokeBulk {
+  private static String extra_jars =
+    System.getProperty("com.cloudera.itest.hivesmoke.IntegrationTestHiveSmokeBulk.extra_jars","");
+
+  static Shell sh = new Shell("/bin/bash -s");
+  static HiveBulkScriptExecutor scripts = new HiveBulkScriptExecutor("scripts/integration");
+
+  private String test;
+
+  public IntegrationTestHiveSmokeBulk(String t) {
+    test = t;
+  }
+
+  @BeforeClass
+  public static void setUp() {
+    def hbase_script = "";
+    def hive_script = "";
+    Shell shHbase = new Shell("hbase shell");
+
+    sh.exec("hive -f ./seed.hql");
+    assertEquals("Can not initialize seed databases",
+                 0, sh.ret);
+
+    ['PARTITION_STAT_TBL', 'countries', 'hbase_pushdown', 'stats_src', 'hbase_part',
+     'hbase_table_0', 'hbase_table_3', 'hbase_table_4', 'hbase_table_6', 'hbase_table_7',
+     'hbase_table_8', 'states', 'users', 'empty_hbase_table'].each { 
+      hbase_script <<= "disable '${it}'\ndrop '${it}'\n";
+      hive_script <<= "drop table ${it};\n";  
+    }
+    shHbase.exec("${hbase_script}\nquit\n\n");
+    sh.exec("hive << __EOT__\n${hive_script}__EOT__");
+  }
+
+  @AfterClass
+  public static void tearDown() {
+    sh.exec("hadoop fs -rmr -skipTrash /user/hive/warehouse",
+            "hadoop fs -rmr -skipTrash /tmp/count");
+  }
+
+  @Parameters
+  public static Map<String, Object[]> readTestCases() {
+    Map res = [:];
+    scripts.getScripts().each { 
+      res[it] = ([it] as String[]);
+    };
+    return res;
+  }
+
+  @Test
+  public void testHiveBulk() {
+    scripts.runScript(test, "${(extra_jars == '') ? '' : '--auxpath '}$extra_jars");
+  }
+}

Added: incubator/bigtop/trunk/test/src/smokes/hive/src/test/groovy/com/cloudera/itest/hivesmoke/TestHiveSmokeBulk.groovy
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/test/src/smokes/hive/src/test/groovy/com/cloudera/itest/hivesmoke/TestHiveSmokeBulk.groovy?rev=1148501&view=auto
==============================================================================
--- incubator/bigtop/trunk/test/src/smokes/hive/src/test/groovy/com/cloudera/itest/hivesmoke/TestHiveSmokeBulk.groovy (added)
+++ incubator/bigtop/trunk/test/src/smokes/hive/src/test/groovy/com/cloudera/itest/hivesmoke/TestHiveSmokeBulk.groovy Tue Jul 19 19:44:48 2011
@@ -0,0 +1,82 @@
+package com.cloudera.itest.hivesmoke
+
+import org.junit.Test
+import org.junit.AfterClass
+import org.junit.Before
+import org.junit.BeforeClass
+import com.cloudera.itest.shell.Shell
+import com.cloudera.itest.junit.OrderedParameterized
+import org.junit.runner.RunWith
+import org.junit.runners.Parameterized.Parameters
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+@RunWith(OrderedParameterized.class)
+public class TestHiveSmokeBulk {
+  static Shell sh = new Shell("/bin/bash -s");
+  static HiveBulkScriptExecutor scripts = new HiveBulkScriptExecutor("scripts/ql");
+
+  private String test;
+
+  public TestHiveSmokeBulk(String t) {
+    test = t;
+  }
+
+  @Before
+  public void cleanUp() {
+    def hive_script = "";
+    Shell shHive = new Shell("hive");
+    ["analyze_srcpart","authorization_part","columntable",
+     "dest1","hbase_part","hbase_pushdown","merge_dynamic_part",
+     "mp","myinput1","nzhang_part14","src_multi1","src_multi2",
+     "srcbucket_mapjoin","srcpart_merge_dp","stats_src","t1",
+     "triples","u_data","union_out", "T1", "T2", "T3", "smb_input1",
+     "smb_input2", "srcbucket_mapjoin_part", "bucketmapjoin_hash_result_1",
+     "bucketmapjoin_hash_result_2", "bucketmapjoin_tmp_result",
+     "srcbucket_mapjoin_part_2"].each { 
+     hive_script <<= "drop table ${it};\n"; 
+    }
+    shHive.exec("${hive_script} quit; \n"); 
+  }
+
+  @BeforeClass
+  public static void setUp() {
+    sh.exec("hive -f ./seed.hql");
+  }
+
+  @AfterClass
+  public static void tearDown() {
+    sh.exec("hadoop fs -rmr -skipTrash /user/hive/warehouse",
+            "hadoop fs -rmr -skipTrash /tmp/count");
+  }
+
+  @Parameters
+  public static Map<String, Object[]> readTestCases() {
+    Map res = [:];
+    scripts.getScripts().collect { 
+      res[it] = ([it] as String[]);
+    };
+    return res;
+  }
+
+  @Test
+  public void testHiveBulk() {
+    scripts.runScript(test);
+  }
+}

Added: incubator/bigtop/trunk/test/src/smokes/hive/src/test/groovy/com/cloudera/itest/hivesmoke/TestJdbcDriver.java
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/test/src/smokes/hive/src/test/groovy/com/cloudera/itest/hivesmoke/TestJdbcDriver.java?rev=1148501&view=auto
==============================================================================
--- incubator/bigtop/trunk/test/src/smokes/hive/src/test/groovy/com/cloudera/itest/hivesmoke/TestJdbcDriver.java (added)
+++ incubator/bigtop/trunk/test/src/smokes/hive/src/test/groovy/com/cloudera/itest/hivesmoke/TestJdbcDriver.java Tue Jul 19 19:44:48 2011
@@ -0,0 +1,148 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.cloudera.itest.hivesmoke;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Date;
+
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertEquals;
+
+import com.cloudera.itest.shell.Shell;
+
+public class TestJdbcDriver {
+
+  public static String driverName = "org.apache.hadoop.hive.jdbc.HiveDriver";
+  public static String hiveserver_url = "jdbc:hive://localhost:10000/default";
+  public static Shell sh = new Shell("/bin/bash -s");
+  public static String testDir = "/tmp/hive-jdbc." + (new Date().getTime());
+  public static String hiveserver_pid;
+  private Connection con;
+
+  @BeforeClass
+    public static void setUp() throws ClassNotFoundException, InterruptedException {
+    Class.forName(driverName);
+    sh.exec("hadoop fs -mkdir " + testDir);
+    assertTrue("Could not create test directory", sh.getRet() == 0);
+    sh.exec("hadoop fs -copyFromLocal a.txt " + testDir + "/a.txt");
+    assertTrue("Could not copy local file to test directory", sh.getRet() == 0);
+    // start hiveserver in background and remember the pid
+    sh.exec("(HIVE_PORT=10000 hive --service hiveserver > /dev/null 2>&1 & echo $! ) 2> /dev/null");
+    hiveserver_pid = sh.getOut().get(0);
+    Thread.sleep(1000); // allow time for hiveserver to be up
+  }
+
+  @Before
+  public void getConnection() throws SQLException {
+    //System.out.println(hiveserver_url);
+    con = DriverManager.getConnection(hiveserver_url, "", "");
+    //System.out.println("JDBC connection is " +
+    //    (con == null ? "not instantiated." : "instantiated."));
+  }
+
+  @After
+  public void closeConnection() throws SQLException {
+    if (con != null)
+      con.close();
+  }
+
+  @AfterClass
+  public static void tearDown() {
+    sh.exec("hadoop fs -rmr -skipTrash " + testDir);
+    sh.exec("kill -9 " + hiveserver_pid);
+  }
+
+  @Test
+  public void testCreate() throws SQLException {
+    Statement stmt = con.createStatement();
+    String tableName = "hive_jdbc_driver_test";
+    stmt.executeQuery("drop table " + tableName);
+    ResultSet res = stmt.executeQuery("create table " + tableName +
+        " (key int, value string)");
+    // show tables
+    String sql = "show tables";
+    //System.out.println("executing: " + sql);
+    res = stmt.executeQuery(sql);
+    boolean tableCreated = false;
+    while (res.next()) {
+      String tab_name = res.getString(1);
+      //System.out.println(tab_name);
+      if (tab_name.equals(tableName))
+        tableCreated = true;
+    }
+    assertTrue("table " + tableName + " does not appear to be created",
+        tableCreated);
+    // describe table
+    sql = "describe " + tableName;
+    //System.out.println("executing: " + sql);
+    res = stmt.executeQuery(sql);
+    List<String> colNames = new ArrayList<String>();
+    List<String> dataTypes = new ArrayList<String>();
+    while (res.next()) {
+      String col_name = res.getString(1);
+      String data_type = res.getString(2);
+      colNames.add(col_name);
+      dataTypes.add(data_type);
+      //System.out.println(col_name + "\t" + data_type);
+    }
+    assertEquals("table should have two columns", 2, colNames.size());
+    assertEquals("key", colNames.get(0));
+    assertEquals("value", colNames.get(1));
+    assertEquals("int", dataTypes.get(0));
+    assertEquals("string", dataTypes.get(1));
+
+    // load data into table
+    String filepath = testDir + "/a.txt"; // this is an hdfs filepath
+    sql = "load data inpath '" + filepath + "' into table " + tableName;
+    //System.out.println("executing: " + sql);
+    res = stmt.executeQuery(sql);
+
+    // select
+    sql = "select * from " + tableName;
+    //System.out.println("executing: " + sql);
+    res = stmt.executeQuery(sql);
+    List<Integer> keys = new ArrayList<Integer>();
+    List<String> values = new ArrayList<String>();
+    while (res.next()) {
+      int key = res.getInt(1);
+      String value = res.getString(2);
+      keys.add(new Integer(key));
+      values.add(value);
+      //System.out.println("" + key + "\t" + value);
+    }
+    assertEquals("table should have two rows", 2, keys.size());
+    assertEquals(new Integer(1), keys.get(0));
+    assertEquals(new Integer(2), keys.get(1));
+    assertEquals("foo", values.get(0));
+    assertEquals("bar", values.get(1));
+  }
+
+}

Added: incubator/bigtop/trunk/test/src/smokes/hive/src/test/resources/a.txt
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/test/src/smokes/hive/src/test/resources/a.txt?rev=1148501&view=auto
==============================================================================
--- incubator/bigtop/trunk/test/src/smokes/hive/src/test/resources/a.txt (added)
+++ incubator/bigtop/trunk/test/src/smokes/hive/src/test/resources/a.txt Tue Jul 19 19:44:48 2011
@@ -0,0 +1,2 @@
+1foo
+2bar

Added: incubator/bigtop/trunk/test/src/smokes/hive/src/test/resources/scripts/integration/hbase_joins/in
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/test/src/smokes/hive/src/test/resources/scripts/integration/hbase_joins/in?rev=1148501&view=auto
==============================================================================
--- incubator/bigtop/trunk/test/src/smokes/hive/src/test/resources/scripts/integration/hbase_joins/in (added)
+++ incubator/bigtop/trunk/test/src/smokes/hive/src/test/resources/scripts/integration/hbase_joins/in Tue Jul 19 19:44:48 2011
@@ -0,0 +1,82 @@
+DROP TABLE users;
+DROP TABLE states;
+DROP TABLE countries;
+DROP TABLE users_level;
+
+-- From HIVE-1257
+
+CREATE TABLE users(key string, state string, country string, country_id int)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = "info:state,info:country,info:country_id"
+);
+
+CREATE TABLE states(key string, name string)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = "state:name"
+);
+
+CREATE TABLE countries(key string, name string, country string, country_id int)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = "info:name,info:country,info:country_id"
+);
+
+INSERT OVERWRITE TABLE users SELECT 'user1', 'IA', 'USA', 0
+FROM src WHERE key=100;
+
+INSERT OVERWRITE TABLE states SELECT 'IA', 'Iowa'
+FROM src WHERE key=100;
+
+INSERT OVERWRITE TABLE countries SELECT 'USA', 'United States', 'USA', 1
+FROM src WHERE key=100;
+
+set hive.input.format = org.apache.hadoop.hive.ql.io.HiveInputFormat;
+
+SELECT u.key, u.country, c.name, c.key FROM users u JOIN countries c 
+ON (u.country = c.key);
+
+SELECT u.key, u.country, c.name, c.key FROM users u JOIN countries c
+ON (u.country = c.country);
+
+SELECT u.key, u.country, c.name, c.key FROM users u JOIN countries c 
+ON (u.country_id = c.country_id);
+
+SELECT u.key, u.state, s.name FROM users u JOIN states s 
+ON (u.state = s.key);
+
+set hive.input.format = org.apache.hadoop.hive.ql.io.CombineHiveInputFormat;
+
+SELECT u.key, u.country, c.name, c.key FROM users u JOIN countries c 
+ON (u.country = c.key);
+
+SELECT u.key, u.country, c.name, c.key FROM users u JOIN countries c
+ON (u.country = c.country);
+
+SELECT u.key, u.country, c.name, c.key FROM users u JOIN countries c 
+ON (u.country_id = c.country_id);
+
+SELECT u.key, u.state, s.name FROM users u JOIN states s 
+ON (u.state = s.key);
+
+DROP TABLE users;
+DROP TABLE states;
+DROP TABLE countries;
+
+CREATE TABLE users(key int, userid int, username string, created int) 
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES ("hbase.columns.mapping" = ":key,f:userid,f:nickname,f:created");
+
+CREATE TABLE users_level(key int, userid int, level int)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES ("hbase.columns.mapping" = ":key,f:userid,f:level");
+
+-- HIVE-1903:  the problem fixed here showed up even without any data,
+-- so no need to load any to test it
+SELECT year(from_unixtime(users.created)) AS year, level, count(users.userid) AS num 
+ FROM users JOIN users_level ON (users.userid = users_level.userid) 
+ GROUP BY year(from_unixtime(users.created)), level;
+
+DROP TABLE users;
+DROP TABLE users_level;

Added: incubator/bigtop/trunk/test/src/smokes/hive/src/test/resources/scripts/integration/hbase_joins/out
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/test/src/smokes/hive/src/test/resources/scripts/integration/hbase_joins/out?rev=1148501&view=auto
==============================================================================
--- incubator/bigtop/trunk/test/src/smokes/hive/src/test/resources/scripts/integration/hbase_joins/out (added)
+++ incubator/bigtop/trunk/test/src/smokes/hive/src/test/resources/scripts/integration/hbase_joins/out Tue Jul 19 19:44:48 2011
@@ -0,0 +1,111 @@
+DROP TABLE users
+
+DROP TABLE states
+
+DROP TABLE countries
+
+DROP TABLE users_level
+
+
+-- From HIVE-1257
+
+CREATE TABLE users(key string, state string, country string, country_id int)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = "info:state,info:country,info:country_id"
+)
+
+
+CREATE TABLE states(key string, name string)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = "state:name"
+)
+
+
+CREATE TABLE countries(key string, name string, country string, country_id int)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = "info:name,info:country,info:country_id"
+)
+
+
+INSERT OVERWRITE TABLE users SELECT 'user1', 'IA', 'USA', 0
+FROM src WHERE key=100
+
+
+INSERT OVERWRITE TABLE states SELECT 'IA', 'Iowa'
+FROM src WHERE key=100
+
+
+INSERT OVERWRITE TABLE countries SELECT 'USA', 'United States', 'USA', 1
+FROM src WHERE key=100
+set hive.input.format = org.apache.hadoop.hive.ql.io.HiveInputFormat
+
+
+SELECT u.key, u.country, c.name, c.key FROM users u JOIN countries c 
+ON (u.country = c.key)
+user1	USA	United States	USA
+
+
+SELECT u.key, u.country, c.name, c.key FROM users u JOIN countries c
+ON (u.country = c.country)
+user1	USA	United States	USA
+
+
+SELECT u.key, u.country, c.name, c.key FROM users u JOIN countries c 
+ON (u.country_id = c.country_id)
+
+
+SELECT u.key, u.state, s.name FROM users u JOIN states s 
+ON (u.state = s.key)
+user1	IA	Iowa
+set hive.input.format = org.apache.hadoop.hive.ql.io.CombineHiveInputFormat
+
+
+SELECT u.key, u.country, c.name, c.key FROM users u JOIN countries c 
+ON (u.country = c.key)
+user1	USA	United States	USA
+
+
+SELECT u.key, u.country, c.name, c.key FROM users u JOIN countries c
+ON (u.country = c.country)
+user1	USA	United States	USA
+
+
+SELECT u.key, u.country, c.name, c.key FROM users u JOIN countries c 
+ON (u.country_id = c.country_id)
+
+
+SELECT u.key, u.state, s.name FROM users u JOIN states s 
+ON (u.state = s.key)
+user1	IA	Iowa
+
+
+DROP TABLE users
+
+DROP TABLE states
+
+DROP TABLE countries
+
+
+CREATE TABLE users(key int, userid int, username string, created int) 
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES ("hbase.columns.mapping" = ":key,f:userid,f:nickname,f:created")
+
+
+CREATE TABLE users_level(key int, userid int, level int)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES ("hbase.columns.mapping" = ":key,f:userid,f:level")
+
+
+-- HIVE-1903:  the problem fixed here showed up even without any data,
+-- so no need to load any to test it
+SELECT year(from_unixtime(users.created)) AS year, level, count(users.userid) AS num 
+ FROM users JOIN users_level ON (users.userid = users_level.userid) 
+ GROUP BY year(from_unixtime(users.created)), level
+
+
+DROP TABLE users
+
+DROP TABLE users_level

Added: incubator/bigtop/trunk/test/src/smokes/hive/src/test/resources/scripts/integration/hbase_pushdown/in
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/test/src/smokes/hive/src/test/resources/scripts/integration/hbase_pushdown/in?rev=1148501&view=auto
==============================================================================
--- incubator/bigtop/trunk/test/src/smokes/hive/src/test/resources/scripts/integration/hbase_pushdown/in (added)
+++ incubator/bigtop/trunk/test/src/smokes/hive/src/test/resources/scripts/integration/hbase_pushdown/in Tue Jul 19 19:44:48 2011
@@ -0,0 +1,53 @@
+CREATE TABLE hbase_pushdown(key int, value string) 
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES ("hbase.columns.mapping" = ":key,cf:string");
+
+INSERT OVERWRITE TABLE hbase_pushdown 
+SELECT *
+FROM src;
+
+-- with full pushdown
+explain select * from hbase_pushdown where key=90;
+
+select * from hbase_pushdown where key=90;
+
+-- with partial pushdown
+
+explain select * from hbase_pushdown where key=90 and value like '%90%';
+
+select * from hbase_pushdown where key=90 and value like '%90%';
+
+-- with two residuals
+
+explain select * from hbase_pushdown
+where key=90 and value like '%90%' and key=cast(value as int);
+
+-- with contradictory pushdowns
+
+explain select * from hbase_pushdown
+where key=80 and key=90 and value like '%90%';
+
+select * from hbase_pushdown
+where key=80 and key=90 and value like '%90%';
+
+-- with nothing to push down
+
+explain select * from hbase_pushdown;
+
+-- with a predicate which is not actually part of the filter, so
+-- it should be ignored by pushdown
+
+explain select * from hbase_pushdown
+where (case when key=90 then 2 else 4 end) > 3;
+
+-- with a predicate which is under an OR, so it should
+-- be ignored by pushdown
+
+explain select * from hbase_pushdown
+where key=80 or value like '%90%';
+
+set hive.optimize.ppd.storage=false;
+
+-- with pushdown disabled
+
+explain select * from hbase_pushdown where key=90;

Added: incubator/bigtop/trunk/test/src/smokes/hive/src/test/resources/scripts/integration/hbase_pushdown/out
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/test/src/smokes/hive/src/test/resources/scripts/integration/hbase_pushdown/out?rev=1148501&view=auto
==============================================================================
--- incubator/bigtop/trunk/test/src/smokes/hive/src/test/resources/scripts/integration/hbase_pushdown/out (added)
+++ incubator/bigtop/trunk/test/src/smokes/hive/src/test/resources/scripts/integration/hbase_pushdown/out Tue Jul 19 19:44:48 2011
@@ -0,0 +1,370 @@
+CREATE TABLE hbase_pushdown(key int, value string) 
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES ("hbase.columns.mapping" = ":key,cf:string")
+
+
+INSERT OVERWRITE TABLE hbase_pushdown 
+SELECT *
+FROM src
+
+
+-- with full pushdown
+explain select * from hbase_pushdown where key=90
+ABSTRACT SYNTAX TREE:
+  (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME hbase_pushdown))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_WHERE (= (TOK_TABLE_OR_COL key) 90))))
+
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 is a root stage
+
+STAGE PLANS:
+  Stage: Stage-1
+    Map Reduce
+      Alias -> Map Operator Tree:
+        hbase_pushdown 
+          TableScan
+            alias: hbase_pushdown
+            filterExpr:
+                expr: (key = 90)
+                type: boolean
+            Filter Operator
+              predicate:
+                  expr: (key = 90)
+                  type: boolean
+              Select Operator
+                expressions:
+                      expr: key
+                      type: int
+                      expr: value
+                      type: string
+                outputColumnNames: _col0, _col1
+                File Output Operator
+                  compressed: false
+                  GlobalTableId: 0
+                  table:
+                      input format: org.apache.hadoop.mapred.TextInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+
+
+
+
+select * from hbase_pushdown where key=90
+90	val_90
+
+
+-- with partial pushdown
+
+explain select * from hbase_pushdown where key=90 and value like '%90%'
+ABSTRACT SYNTAX TREE:
+  (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME hbase_pushdown))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_WHERE (and (= (TOK_TABLE_OR_COL key) 90) (like (TOK_TABLE_OR_COL value) '%90%')))))
+
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 is a root stage
+
+STAGE PLANS:
+  Stage: Stage-1
+    Map Reduce
+      Alias -> Map Operator Tree:
+        hbase_pushdown 
+          TableScan
+            alias: hbase_pushdown
+            filterExpr:
+                expr: (key = 90)
+                type: boolean
+            Filter Operator
+              predicate:
+                  expr: (value like '%90%')
+                  type: boolean
+              Filter Operator
+                predicate:
+                    expr: ((key = 90) and (value like '%90%'))
+                    type: boolean
+                Select Operator
+                  expressions:
+                        expr: key
+                        type: int
+                        expr: value
+                        type: string
+                  outputColumnNames: _col0, _col1
+                  File Output Operator
+                    compressed: false
+                    GlobalTableId: 0
+                    table:
+                        input format: org.apache.hadoop.mapred.TextInputFormat
+                        output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+
+
+
+
+select * from hbase_pushdown where key=90 and value like '%90%'
+90	val_90
+
+
+-- with two residuals
+
+explain select * from hbase_pushdown
+where key=90 and value like '%90%' and key=cast(value as int)
+ABSTRACT SYNTAX TREE:
+  (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME hbase_pushdown))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_WHERE (and (and (= (TOK_TABLE_OR_COL key) 90) (like (TOK_TABLE_OR_COL value) '%90%')) (= (TOK_TABLE_OR_COL key) (TOK_FUNCTION TOK_INT (TOK_TABLE_OR_COL value)))))))
+
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 is a root stage
+
+STAGE PLANS:
+  Stage: Stage-1
+    Map Reduce
+      Alias -> Map Operator Tree:
+        hbase_pushdown 
+          TableScan
+            alias: hbase_pushdown
+            filterExpr:
+                expr: (key = 90)
+                type: boolean
+            Filter Operator
+              predicate:
+                  expr: ((value like '%90%') and (key = UDFToInteger(value)))
+                  type: boolean
+              Filter Operator
+                predicate:
+                    expr: (((key = 90) and (value like '%90%')) and (key = UDFToInteger(value)))
+                    type: boolean
+                Select Operator
+                  expressions:
+                        expr: key
+                        type: int
+                        expr: value
+                        type: string
+                  outputColumnNames: _col0, _col1
+                  File Output Operator
+                    compressed: false
+                    GlobalTableId: 0
+                    table:
+                        input format: org.apache.hadoop.mapred.TextInputFormat
+                        output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+
+
+
+
+-- with contradictory pushdowns
+
+explain select * from hbase_pushdown
+where key=80 and key=90 and value like '%90%'
+ABSTRACT SYNTAX TREE:
+  (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME hbase_pushdown))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_WHERE (and (and (= (TOK_TABLE_OR_COL key) 80) (= (TOK_TABLE_OR_COL key) 90)) (like (TOK_TABLE_OR_COL value) '%90%')))))
+
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 is a root stage
+
+STAGE PLANS:
+  Stage: Stage-1
+    Map Reduce
+      Alias -> Map Operator Tree:
+        hbase_pushdown 
+          TableScan
+            alias: hbase_pushdown
+            Filter Operator
+              predicate:
+                  expr: (((key = 80) and (key = 90)) and (value like '%90%'))
+                  type: boolean
+              Filter Operator
+                predicate:
+                    expr: (((key = 80) and (key = 90)) and (value like '%90%'))
+                    type: boolean
+                Select Operator
+                  expressions:
+                        expr: key
+                        type: int
+                        expr: value
+                        type: string
+                  outputColumnNames: _col0, _col1
+                  File Output Operator
+                    compressed: false
+                    GlobalTableId: 0
+                    table:
+                        input format: org.apache.hadoop.mapred.TextInputFormat
+                        output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+
+
+
+
+select * from hbase_pushdown
+where key=80 and key=90 and value like '%90%'
+
+
+-- with nothing to push down
+
+explain select * from hbase_pushdown
+ABSTRACT SYNTAX TREE:
+  (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME hbase_pushdown))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF))))
+
+STAGE DEPENDENCIES:
+  Stage-0 is a root stage
+
+STAGE PLANS:
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+
+
+
+
+-- with a predicate which is not actually part of the filter, so
+-- it should be ignored by pushdown
+
+explain select * from hbase_pushdown
+where (case when key=90 then 2 else 4 end) > 3
+ABSTRACT SYNTAX TREE:
+  (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME hbase_pushdown))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_WHERE (> (TOK_FUNCTION when (= (TOK_TABLE_OR_COL key) 90) 2 4) 3))))
+
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 is a root stage
+
+STAGE PLANS:
+  Stage: Stage-1
+    Map Reduce
+      Alias -> Map Operator Tree:
+        hbase_pushdown 
+          TableScan
+            alias: hbase_pushdown
+            Filter Operator
+              predicate:
+                  expr: (CASE WHEN ((key = 90)) THEN (2) ELSE (4) END > 3)
+                  type: boolean
+              Filter Operator
+                predicate:
+                    expr: (CASE WHEN ((key = 90)) THEN (2) ELSE (4) END > 3)
+                    type: boolean
+                Select Operator
+                  expressions:
+                        expr: key
+                        type: int
+                        expr: value
+                        type: string
+                  outputColumnNames: _col0, _col1
+                  File Output Operator
+                    compressed: false
+                    GlobalTableId: 0
+                    table:
+                        input format: org.apache.hadoop.mapred.TextInputFormat
+                        output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+
+
+
+
+-- with a predicate which is under an OR, so it should
+-- be ignored by pushdown
+
+explain select * from hbase_pushdown
+where key=80 or value like '%90%'
+ABSTRACT SYNTAX TREE:
+  (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME hbase_pushdown))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_WHERE (or (= (TOK_TABLE_OR_COL key) 80) (like (TOK_TABLE_OR_COL value) '%90%')))))
+
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 is a root stage
+
+STAGE PLANS:
+  Stage: Stage-1
+    Map Reduce
+      Alias -> Map Operator Tree:
+        hbase_pushdown 
+          TableScan
+            alias: hbase_pushdown
+            Filter Operator
+              predicate:
+                  expr: ((key = 80) or (value like '%90%'))
+                  type: boolean
+              Filter Operator
+                predicate:
+                    expr: ((key = 80) or (value like '%90%'))
+                    type: boolean
+                Select Operator
+                  expressions:
+                        expr: key
+                        type: int
+                        expr: value
+                        type: string
+                  outputColumnNames: _col0, _col1
+                  File Output Operator
+                    compressed: false
+                    GlobalTableId: 0
+                    table:
+                        input format: org.apache.hadoop.mapred.TextInputFormat
+                        output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+
+
+set hive.optimize.ppd.storage=false
+
+
+-- with pushdown disabled
+
+explain select * from hbase_pushdown where key=90
+ABSTRACT SYNTAX TREE:
+  (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME hbase_pushdown))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_WHERE (= (TOK_TABLE_OR_COL key) 90))))
+
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 is a root stage
+
+STAGE PLANS:
+  Stage: Stage-1
+    Map Reduce
+      Alias -> Map Operator Tree:
+        hbase_pushdown 
+          TableScan
+            alias: hbase_pushdown
+            Filter Operator
+              predicate:
+                  expr: (key = 90)
+                  type: boolean
+              Filter Operator
+                predicate:
+                    expr: (key = 90)
+                    type: boolean
+                Select Operator
+                  expressions:
+                        expr: key
+                        type: int
+                        expr: value
+                        type: string
+                  outputColumnNames: _col0, _col1
+                  File Output Operator
+                    compressed: false
+                    GlobalTableId: 0
+                    table:
+                        input format: org.apache.hadoop.mapred.TextInputFormat
+                        output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+
+

Added: incubator/bigtop/trunk/test/src/smokes/hive/src/test/resources/scripts/integration/hbase_queries/filter
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/test/src/smokes/hive/src/test/resources/scripts/integration/hbase_queries/filter?rev=1148501&view=auto
==============================================================================
--- incubator/bigtop/trunk/test/src/smokes/hive/src/test/resources/scripts/integration/hbase_queries/filter (added)
+++ incubator/bigtop/trunk/test/src/smokes/hive/src/test/resources/scripts/integration/hbase_queries/filter Tue Jul 19 19:44:48 2011
@@ -0,0 +1,8 @@
+sed -e 's#hdfs://[^/]*/#HDFS_URL/#' \
+    -e 's#createTime:[0-9][0-9]*#createTime:NOW#g' \
+    -e 's#transient_lastDdlTime=[0-9][0-9]*#transient_lastDdlTime=NOW#g' \
+    -e '/Map Operator Tree:/,/Reduce Output Operator/d' \
+    -e '/^  (TOK_QUERY/d' \
+    -e '/Detailed Table Information/s#owner:[^,]*,#owner:OWNER,#' \
+    -e 's#name: default.hbase_table_3#name: HBASE_TABLE#' \
+    -e 's#name: hbase_table_3#name: HBASE_TABLE#'

Added: incubator/bigtop/trunk/test/src/smokes/hive/src/test/resources/scripts/integration/hbase_queries/in
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/test/src/smokes/hive/src/test/resources/scripts/integration/hbase_queries/in?rev=1148501&view=auto
==============================================================================
--- incubator/bigtop/trunk/test/src/smokes/hive/src/test/resources/scripts/integration/hbase_queries/in (added)
+++ incubator/bigtop/trunk/test/src/smokes/hive/src/test/resources/scripts/integration/hbase_queries/in Tue Jul 19 19:44:48 2011
@@ -0,0 +1,160 @@
+DROP TABLE hbase_table_1;
+CREATE TABLE hbase_table_1(key int, value string) 
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES ("hbase.columns.mapping" = "cf:string")
+TBLPROPERTIES ("hbase.table.name" = "hbase_table_0");
+
+DESCRIBE EXTENDED hbase_table_1;
+
+select * from hbase_table_1;
+
+EXPLAIN FROM src INSERT OVERWRITE TABLE hbase_table_1 SELECT * WHERE (key%2)=0;
+FROM src INSERT OVERWRITE TABLE hbase_table_1 SELECT * WHERE (key%2)=0;
+
+DROP TABLE hbase_table_2;
+CREATE EXTERNAL TABLE hbase_table_2(key int, value string) 
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES ("hbase.columns.mapping" = "cf:string")
+TBLPROPERTIES ("hbase.table.name" = "hbase_table_0");
+
+EXPLAIN 
+SELECT Y.* 
+FROM 
+(SELECT hbase_table_1.* FROM hbase_table_1) x
+JOIN 
+(SELECT src.* FROM src) Y
+ON (x.key = Y.key)
+ORDER BY key, value LIMIT 20;
+
+SELECT Y.* 
+FROM 
+(SELECT hbase_table_1.* FROM hbase_table_1) x
+JOIN 
+(SELECT src.* FROM src) Y
+ON (x.key = Y.key)
+ORDER BY key, value LIMIT 20;
+
+EXPLAIN 
+SELECT Y.*
+FROM 
+(SELECT hbase_table_1.* FROM hbase_table_1 WHERE hbase_table_1.key > 100) x
+JOIN 
+(SELECT hbase_table_2.* FROM hbase_table_2 WHERE hbase_table_2.key < 120) Y
+ON (x.key = Y.key)
+ORDER BY key, value;
+
+SELECT Y.*
+FROM 
+(SELECT hbase_table_1.* FROM hbase_table_1 WHERE hbase_table_1.key > 100) x
+JOIN 
+(SELECT hbase_table_2.* FROM hbase_table_2 WHERE hbase_table_2.key < 120) Y
+ON (x.key = Y.key)
+ORDER BY key,value;
+
+DROP TABLE empty_hbase_table;
+CREATE TABLE empty_hbase_table(key int, value string) 
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES ("hbase.columns.mapping" = "cf:string");
+
+DROP TABLE empty_normal_table;
+CREATE TABLE empty_normal_table(key int, value string);
+
+select * from (select count(1) as c from empty_normal_table union all select count(1) as c from empty_hbase_table) x order by c;
+select * from (select count(1) c from empty_normal_table union all select count(1) as c from hbase_table_1) x order by c;
+select * from (select count(1) c from src union all select count(1) as c from empty_hbase_table) x order by c;
+select * from (select count(1) c from src union all select count(1) as c from hbase_table_1) x order by c;
+
+CREATE TABLE hbase_table_3(key int, value string, count int) 
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = "cf:val,cf2:count"
+);
+
+EXPLAIN 
+INSERT OVERWRITE TABLE hbase_table_3
+SELECT x.key, x.value, Y.count 
+FROM 
+(SELECT hbase_table_1.* FROM hbase_table_1) x
+JOIN 
+(SELECT src.key, count(src.key) as count FROM src GROUP BY src.key) Y
+ON (x.key = Y.key);
+
+INSERT OVERWRITE TABLE hbase_table_3
+SELECT x.key, x.value, Y.count 
+FROM 
+(SELECT hbase_table_1.* FROM hbase_table_1) x
+JOIN 
+(SELECT src.key, count(src.key) as count FROM src GROUP BY src.key) Y
+ON (x.key = Y.key);
+
+select count(1) from hbase_table_3;
+select * from hbase_table_3 order by key, value limit 5;
+select key, count from hbase_table_3 order by key, count desc limit 5;
+
+DROP TABLE hbase_table_4;
+CREATE TABLE hbase_table_4(key int, value1 string, value2 int, value3 int) 
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = "a:b,a:c,d:e"
+);
+
+INSERT OVERWRITE TABLE hbase_table_4 SELECT key, value, key+1, key+2 
+FROM src WHERE key=98 OR key=100;
+
+SELECT * FROM hbase_table_4 ORDER BY key;
+
+DROP TABLE hbase_table_5;
+CREATE EXTERNAL TABLE hbase_table_5(key int, value map<string,string>) 
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES ("hbase.columns.mapping" = "a:")
+TBLPROPERTIES ("hbase.table.name" = "hbase_table_4");
+
+SELECT * FROM hbase_table_5 ORDER BY key;
+
+DROP TABLE hbase_table_6;
+CREATE TABLE hbase_table_6(key int, value map<string,string>) 
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = ":key,cf:"
+);
+INSERT OVERWRITE TABLE hbase_table_6 SELECT key, map(value, key) FROM src
+WHERE key=98 OR key=100;
+
+SELECT * FROM hbase_table_6 ORDER BY key;
+
+DROP TABLE hbase_table_7;
+CREATE TABLE hbase_table_7(value map<string,string>, key int) 
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = "cf:,:key"
+);
+INSERT OVERWRITE TABLE hbase_table_7 
+SELECT map(value, key, upper(value), key+1), key FROM src
+WHERE key=98 OR key=100;
+
+SELECT * FROM hbase_table_7 ORDER BY key;
+
+set hive.hbase.wal.enabled=false;
+
+DROP TABLE hbase_table_8;
+CREATE TABLE hbase_table_8(key int, value1 string, value2 int, value3 int) 
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = "a:b,a:c,d:e"
+);
+
+INSERT OVERWRITE TABLE hbase_table_8 SELECT key, value, key+1, key+2 
+FROM src WHERE key=98 OR key=100;
+
+SELECT * FROM hbase_table_8 ORDER BY key;
+
+DROP TABLE hbase_table_1;
+DROP TABLE hbase_table_2;
+DROP TABLE hbase_table_3;
+DROP TABLE hbase_table_4;
+DROP TABLE hbase_table_5;
+DROP TABLE hbase_table_6;
+DROP TABLE hbase_table_7;
+DROP TABLE hbase_table_8;
+DROP TABLE empty_hbase_table;
+DROP TABLE empty_normal_table;

Added: incubator/bigtop/trunk/test/src/smokes/hive/src/test/resources/scripts/integration/hbase_queries/out
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/test/src/smokes/hive/src/test/resources/scripts/integration/hbase_queries/out?rev=1148501&view=auto
==============================================================================
--- incubator/bigtop/trunk/test/src/smokes/hive/src/test/resources/scripts/integration/hbase_queries/out (added)
+++ incubator/bigtop/trunk/test/src/smokes/hive/src/test/resources/scripts/integration/hbase_queries/out Tue Jul 19 19:44:48 2011
@@ -0,0 +1,687 @@
+DROP TABLE hbase_table_1
+
+CREATE TABLE hbase_table_1(key int, value string) 
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES ("hbase.columns.mapping" = "cf:string")
+TBLPROPERTIES ("hbase.table.name" = "hbase_table_0")
+
+
+DESCRIBE EXTENDED hbase_table_1
+key	int	from deserializer
+value	string	from deserializer
+	 	 
+Detailed Table Information	Table(tableName:hbase_table_1, dbName:default, owner:testuser1@MINOTAUR.CLOUDERA.COM, createTime:1301900428, lastAccessTime:0, retention:0, sd:StorageDescriptor(cols:[FieldSchema(name:key, type:int, comment:null), FieldSchema(name:value, type:string, comment:null)], location:hdfs://minotaur01.sf.cloudera.com:17020/user/hive/warehouse/hbase_table_1, inputFormat:org.apache.hadoop.hive.hbase.HiveHBaseTableInputFormat, outputFormat:org.apache.hadoop.hive.hbase.HiveHBaseTableOutputFormat, compressed:false, numBuckets:-1, serdeInfo:SerDeInfo(name:null, serializationLib:org.apache.hadoop.hive.hbase.HBaseSerDe, parameters:{serialization.format=1, hbase.columns.mapping=cf:string}), bucketCols:[], sortCols:[], parameters:{}), partitionKeys:[], parameters:{hbase.table.name=hbase_table_0, transient_lastDdlTime=1301900428, storage_handler=org.apache.hadoop.hive.hbase.HBaseStorageHandler}, viewOriginalText:null, viewExpandedText:null, tableType:MANAGED_TABLE)	
+
+
+select * from hbase_table_1
+
+
+EXPLAIN FROM src INSERT OVERWRITE TABLE hbase_table_1 SELECT * WHERE (key%2)=0
+ABSTRACT SYNTAX TREE:
+  (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME hbase_table_1))) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_WHERE (= (% (TOK_TABLE_OR_COL key) 2) 0))))
+
+STAGE DEPENDENCIES:
+  Stage-0 is a root stage
+
+STAGE PLANS:
+  Stage: Stage-0
+    Map Reduce
+      Alias -> Map Operator Tree:
+        src 
+          TableScan
+            alias: src
+            Filter Operator
+              predicate:
+                  expr: ((key % 2) = 0)
+                  type: boolean
+              Filter Operator
+                predicate:
+                    expr: ((key % 2) = 0)
+                    type: boolean
+                Select Operator
+                  expressions:
+                        expr: key
+                        type: string
+                        expr: value
+                        type: string
+                  outputColumnNames: _col0, _col1
+                  Select Operator
+                    expressions:
+                          expr: UDFToInteger(_col0)
+                          type: int
+                          expr: _col1
+                          type: string
+                    outputColumnNames: _col0, _col1
+                    File Output Operator
+                      compressed: false
+                      GlobalTableId: 1
+                      table:
+                          input format: org.apache.hadoop.hive.hbase.HiveHBaseTableInputFormat
+                          output format: org.apache.hadoop.hive.hbase.HiveHBaseTableOutputFormat
+                          serde: org.apache.hadoop.hive.hbase.HBaseSerDe
+                          name: default.hbase_table_1
+
+
+
+FROM src INSERT OVERWRITE TABLE hbase_table_1 SELECT * WHERE (key%2)=0
+
+
+DROP TABLE hbase_table_2
+
+CREATE EXTERNAL TABLE hbase_table_2(key int, value string) 
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES ("hbase.columns.mapping" = "cf:string")
+TBLPROPERTIES ("hbase.table.name" = "hbase_table_0")
+
+
+EXPLAIN 
+SELECT Y.* 
+FROM 
+(SELECT hbase_table_1.* FROM hbase_table_1) x
+JOIN 
+(SELECT src.* FROM src) Y
+ON (x.key = Y.key)
+ORDER BY key, value LIMIT 20
+ABSTRACT SYNTAX TREE:
+  (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME hbase_table_1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF (TOK_TABNAME hbase_table_1)))))) x) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF (TOK_TABNAME src)))))) Y) (= (. (TOK_TABLE_OR_COL x) key) (. (TOK_TABLE_OR_COL Y) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF (TOK_TABNAME Y)))) (TOK_ORDERBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL key)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL value))) (TOK_LIMIT 20)))
+
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-2 depends on stages: Stage-1
+  Stage-0 is a root stage
+
+STAGE PLANS:
+  Stage: Stage-1
+    Map Reduce
+      Alias -> Map Operator Tree:
+        x:hbase_table_1 
+          TableScan
+            alias: hbase_table_1
+            Select Operator
+              expressions:
+                    expr: key
+                    type: int
+              outputColumnNames: _col0
+              Reduce Output Operator
+                key expressions:
+                      expr: UDFToDouble(_col0)
+                      type: double
+                sort order: +
+                Map-reduce partition columns:
+                      expr: UDFToDouble(_col0)
+                      type: double
+                tag: 0
+        y:src 
+          TableScan
+            alias: src
+            Select Operator
+              expressions:
+                    expr: key
+                    type: string
+                    expr: value
+                    type: string
+              outputColumnNames: _col0, _col1
+              Reduce Output Operator
+                key expressions:
+                      expr: UDFToDouble(_col0)
+                      type: double
+                sort order: +
+                Map-reduce partition columns:
+                      expr: UDFToDouble(_col0)
+                      type: double
+                tag: 1
+                value expressions:
+                      expr: _col0
+                      type: string
+                      expr: _col1
+                      type: string
+      Reduce Operator Tree:
+        Join Operator
+          condition map:
+               Inner Join 0 to 1
+          condition expressions:
+            0 
+            1 {VALUE._col0} {VALUE._col1}
+          handleSkewJoin: false
+          outputColumnNames: _col2, _col3
+          Select Operator
+            expressions:
+                  expr: _col2
+                  type: string
+                  expr: _col3
+                  type: string
+            outputColumnNames: _col0, _col1
+            File Output Operator
+              compressed: false
+              GlobalTableId: 0
+              table:
+                  input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                  output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+  Stage: Stage-2
+    Map Reduce
+      Alias -> Map Operator Tree:
+        hdfs://minotaur01.sf.cloudera.com:17020/tmp/hive-testuser1/hive_2011-04-04_00-00-37_447_311787050586995300/-mr-10002 
+            Reduce Output Operator
+              key expressions:
+                    expr: _col0
+                    type: string
+                    expr: _col1
+                    type: string
+              sort order: ++
+              tag: -1
+              value expressions:
+                    expr: _col0
+                    type: string
+                    expr: _col1
+                    type: string
+      Reduce Operator Tree:
+        Extract
+          Limit
+            File Output Operator
+              compressed: false
+              GlobalTableId: 0
+              table:
+                  input format: org.apache.hadoop.mapred.TextInputFormat
+                  output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: 20
+
+
+
+
+SELECT Y.* 
+FROM 
+(SELECT hbase_table_1.* FROM hbase_table_1) x
+JOIN 
+(SELECT src.* FROM src) Y
+ON (x.key = Y.key)
+ORDER BY key, value LIMIT 20
+0	val_0
+0	val_0
+0	val_0
+10	val_10
+100	val_100
+100	val_100
+104	val_104
+104	val_104
+114	val_114
+116	val_116
+118	val_118
+118	val_118
+12	val_12
+12	val_12
+120	val_120
+120	val_120
+126	val_126
+128	val_128
+128	val_128
+128	val_128
+
+
+EXPLAIN 
+SELECT Y.*
+FROM 
+(SELECT hbase_table_1.* FROM hbase_table_1 WHERE hbase_table_1.key > 100) x
+JOIN 
+(SELECT hbase_table_2.* FROM hbase_table_2 WHERE hbase_table_2.key < 120) Y
+ON (x.key = Y.key)
+ORDER BY key, value
+ABSTRACT SYNTAX TREE:
+  (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME hbase_table_1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF (TOK_TABNAME hbase_table_1)))) (TOK_WHERE (> (. (TOK_TABLE_OR_COL hbase_table_1) key) 100)))) x) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME hbase_table_2))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF (TOK_TABNAME hbase_table_2)))) (TOK_WHERE (< (. (TOK_TABLE_OR_COL hbase_table_2) key) 120)))) Y) (= (. (TOK_TABLE_OR_COL x) key) (. (TOK_TABLE_OR_COL Y) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF (TOK_TABNAME Y)))) (TOK_ORDERBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL key)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL value)))))
+
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-2 depends on stages: Stage-1
+  Stage-0 is a root stage
+
+STAGE PLANS:
+  Stage: Stage-1
+    Map Reduce
+      Alias -> Map Operator Tree:
+        x:hbase_table_1 
+          TableScan
+            alias: hbase_table_1
+            Filter Operator
+              predicate:
+                  expr: (key > 100)
+                  type: boolean
+              Filter Operator
+                predicate:
+                    expr: (key > 100)
+                    type: boolean
+                Select Operator
+                  expressions:
+                        expr: key
+                        type: int
+                  outputColumnNames: _col0
+                  Reduce Output Operator
+                    key expressions:
+                          expr: _col0
+                          type: int
+                    sort order: +
+                    Map-reduce partition columns:
+                          expr: _col0
+                          type: int
+                    tag: 0
+        y:hbase_table_2 
+          TableScan
+            alias: hbase_table_2
+            Filter Operator
+              predicate:
+                  expr: (key < 120)
+                  type: boolean
+              Filter Operator
+                predicate:
+                    expr: (key < 120)
+                    type: boolean
+                Select Operator
+                  expressions:
+                        expr: key
+                        type: int
+                        expr: value
+                        type: string
+                  outputColumnNames: _col0, _col1
+                  Reduce Output Operator
+                    key expressions:
+                          expr: _col0
+                          type: int
+                    sort order: +
+                    Map-reduce partition columns:
+                          expr: _col0
+                          type: int
+                    tag: 1
+                    value expressions:
+                          expr: _col0
+                          type: int
+                          expr: _col1
+                          type: string
+      Reduce Operator Tree:
+        Join Operator
+          condition map:
+               Inner Join 0 to 1
+          condition expressions:
+            0 
+            1 {VALUE._col0} {VALUE._col1}
+          handleSkewJoin: false
+          outputColumnNames: _col2, _col3
+          Select Operator
+            expressions:
+                  expr: _col2
+                  type: int
+                  expr: _col3
+                  type: string
+            outputColumnNames: _col0, _col1
+            File Output Operator
+              compressed: false
+              GlobalTableId: 0
+              table:
+                  input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                  output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+  Stage: Stage-2
+    Map Reduce
+      Alias -> Map Operator Tree:
+        hdfs://minotaur01.sf.cloudera.com:17020/tmp/hive-testuser1/hive_2011-04-04_00-01-03_920_5397268077686778739/-mr-10002 
+            Reduce Output Operator
+              key expressions:
+                    expr: _col0
+                    type: int
+                    expr: _col1
+                    type: string
+              sort order: ++
+              tag: -1
+              value expressions:
+                    expr: _col0
+                    type: int
+                    expr: _col1
+                    type: string
+      Reduce Operator Tree:
+        Extract
+          File Output Operator
+            compressed: false
+            GlobalTableId: 0
+            table:
+                input format: org.apache.hadoop.mapred.TextInputFormat
+                output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+
+
+
+
+SELECT Y.*
+FROM 
+(SELECT hbase_table_1.* FROM hbase_table_1 WHERE hbase_table_1.key > 100) x
+JOIN 
+(SELECT hbase_table_2.* FROM hbase_table_2 WHERE hbase_table_2.key < 120) Y
+ON (x.key = Y.key)
+ORDER BY key,value
+104	val_104
+114	val_114
+116	val_116
+118	val_118
+
+
+DROP TABLE empty_hbase_table
+
+CREATE TABLE empty_hbase_table(key int, value string) 
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES ("hbase.columns.mapping" = "cf:string")
+
+
+DROP TABLE empty_normal_table
+
+CREATE TABLE empty_normal_table(key int, value string)
+
+
+select * from (select count(1) as c from empty_normal_table union all select count(1) as c from empty_hbase_table) x order by c
+0
+0
+
+select * from (select count(1) c from empty_normal_table union all select count(1) as c from hbase_table_1) x order by c
+0
+155
+
+select * from (select count(1) c from src union all select count(1) as c from empty_hbase_table) x order by c
+0
+500
+
+select * from (select count(1) c from src union all select count(1) as c from hbase_table_1) x order by c
+155
+500
+
+
+CREATE TABLE hbase_table_3(key int, value string, count int) 
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = "cf:val,cf2:count"
+)
+
+
+EXPLAIN 
+INSERT OVERWRITE TABLE hbase_table_3
+SELECT x.key, x.value, Y.count 
+FROM 
+(SELECT hbase_table_1.* FROM hbase_table_1) x
+JOIN 
+(SELECT src.key, count(src.key) as count FROM src GROUP BY src.key) Y
+ON (x.key = Y.key)
+ABSTRACT SYNTAX TREE:
+  (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME hbase_table_1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF (TOK_TABNAME hbase_table_1)))))) x) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL src) key)) (TOK_SELEXPR (TOK_FUNCTION count (. (TOK_TABLE_OR_COL src) key)) count)) (TOK_GROUPBY (. (TOK_TABLE_OR_COL src) key)))) Y) (= (. (TOK_TABLE_OR_COL x) key) (. (TOK_TABLE_OR_COL Y) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME hbase_table_3))) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL x) key)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL x) value)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL Y) count)))))
+
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+  Stage: Stage-1
+    Map Reduce
+      Alias -> Map Operator Tree:
+        y:src 
+          TableScan
+            alias: src
+            Select Operator
+              expressions:
+                    expr: key
+                    type: string
+              outputColumnNames: key
+              Group By Operator
+                aggregations:
+                      expr: count(key)
+                bucketGroup: false
+                keys:
+                      expr: key
+                      type: string
+                mode: hash
+                outputColumnNames: _col0, _col1
+                Reduce Output Operator
+                  key expressions:
+                        expr: _col0
+                        type: string
+                  sort order: +
+                  Map-reduce partition columns:
+                        expr: _col0
+                        type: string
+                  tag: -1
+                  value expressions:
+                        expr: _col1
+                        type: bigint
+      Reduce Operator Tree:
+        Group By Operator
+          aggregations:
+                expr: count(VALUE._col0)
+          bucketGroup: false
+          keys:
+                expr: KEY._col0
+                type: string
+          mode: mergepartial
+          outputColumnNames: _col0, _col1
+          Select Operator
+            expressions:
+                  expr: _col0
+                  type: string
+                  expr: _col1
+                  type: bigint
+            outputColumnNames: _col0, _col1
+            File Output Operator
+              compressed: false
+              GlobalTableId: 0
+              table:
+                  input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                  output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+  Stage: Stage-0
+    Map Reduce
+      Alias -> Map Operator Tree:
+        $INTNAME 
+            Reduce Output Operator
+              key expressions:
+                    expr: UDFToDouble(_col0)
+                    type: double
+              sort order: +
+              Map-reduce partition columns:
+                    expr: UDFToDouble(_col0)
+                    type: double
+              tag: 1
+              value expressions:
+                    expr: _col1
+                    type: bigint
+        x:hbase_table_1 
+          TableScan
+            alias: hbase_table_1
+            Select Operator
+              expressions:
+                    expr: key
+                    type: int
+                    expr: value
+                    type: string
+              outputColumnNames: _col0, _col1
+              Reduce Output Operator
+                key expressions:
+                      expr: UDFToDouble(_col0)
+                      type: double
+                sort order: +
+                Map-reduce partition columns:
+                      expr: UDFToDouble(_col0)
+                      type: double
+                tag: 0
+                value expressions:
+                      expr: _col0
+                      type: int
+                      expr: _col1
+                      type: string
+      Reduce Operator Tree:
+        Join Operator
+          condition map:
+               Inner Join 0 to 1
+          condition expressions:
+            0 {VALUE._col0} {VALUE._col1}
+            1 {VALUE._col1}
+          handleSkewJoin: false
+          outputColumnNames: _col0, _col1, _col3
+          Select Operator
+            expressions:
+                  expr: _col0
+                  type: int
+                  expr: _col1
+                  type: string
+                  expr: _col3
+                  type: bigint
+            outputColumnNames: _col0, _col1, _col2
+            Select Operator
+              expressions:
+                    expr: _col0
+                    type: int
+                    expr: _col1
+                    type: string
+                    expr: UDFToInteger(_col2)
+                    type: int
+              outputColumnNames: _col0, _col1, _col2
+              File Output Operator
+                compressed: false
+                GlobalTableId: 1
+                table:
+                    input format: org.apache.hadoop.hive.hbase.HiveHBaseTableInputFormat
+                    output format: org.apache.hadoop.hive.hbase.HiveHBaseTableOutputFormat
+                    serde: org.apache.hadoop.hive.hbase.HBaseSerDe
+                    name: default.hbase_table_3
+
+
+
+
+INSERT OVERWRITE TABLE hbase_table_3
+SELECT x.key, x.value, Y.count 
+FROM 
+(SELECT hbase_table_1.* FROM hbase_table_1) x
+JOIN 
+(SELECT src.key, count(src.key) as count FROM src GROUP BY src.key) Y
+ON (x.key = Y.key)
+
+
+select count(1) from hbase_table_3
+155
+
+select * from hbase_table_3 order by key, value limit 5
+0	val_0	3
+2	val_2	1
+4	val_4	1
+8	val_8	1
+10	val_10	1
+
+select key, count from hbase_table_3 order by key, count desc limit 5
+0	3
+2	1
+4	1
+8	1
+10	1
+
+
+DROP TABLE hbase_table_4
+
+CREATE TABLE hbase_table_4(key int, value1 string, value2 int, value3 int) 
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = "a:b,a:c,d:e"
+)
+
+
+INSERT OVERWRITE TABLE hbase_table_4 SELECT key, value, key+1, key+2 
+FROM src WHERE key=98 OR key=100
+
+
+SELECT * FROM hbase_table_4 ORDER BY key
+98	val_98	99	100
+100	val_100	101	102
+
+
+DROP TABLE hbase_table_5
+
+CREATE EXTERNAL TABLE hbase_table_5(key int, value map<string,string>) 
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES ("hbase.columns.mapping" = "a:")
+TBLPROPERTIES ("hbase.table.name" = "hbase_table_4")
+
+
+SELECT * FROM hbase_table_5 ORDER BY key
+98	{"b":"val_98","c":"99"}
+100	{"b":"val_100","c":"101"}
+
+
+DROP TABLE hbase_table_6
+
+CREATE TABLE hbase_table_6(key int, value map<string,string>) 
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = ":key,cf:"
+)
+
+INSERT OVERWRITE TABLE hbase_table_6 SELECT key, map(value, key) FROM src
+WHERE key=98 OR key=100
+
+
+SELECT * FROM hbase_table_6 ORDER BY key
+98	{"val_98":"98"}
+100	{"val_100":"100"}
+
+
+DROP TABLE hbase_table_7
+
+CREATE TABLE hbase_table_7(value map<string,string>, key int) 
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = "cf:,:key"
+)
+
+INSERT OVERWRITE TABLE hbase_table_7 
+SELECT map(value, key, upper(value), key+1), key FROM src
+WHERE key=98 OR key=100
+
+
+SELECT * FROM hbase_table_7 ORDER BY key
+{"VAL_98":"99.0","val_98":"98"}	98
+{"VAL_100":"101.0","val_100":"100"}	100
+set hive.hbase.wal.enabled=false
+
+
+DROP TABLE hbase_table_8
+
+CREATE TABLE hbase_table_8(key int, value1 string, value2 int, value3 int) 
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = "a:b,a:c,d:e"
+)
+
+
+INSERT OVERWRITE TABLE hbase_table_8 SELECT key, value, key+1, key+2 
+FROM src WHERE key=98 OR key=100
+
+
+SELECT * FROM hbase_table_8 ORDER BY key
+98	val_98	99	100
+100	val_100	101	102
+
+
+DROP TABLE hbase_table_1
+
+DROP TABLE hbase_table_2
+
+DROP TABLE hbase_table_3
+
+DROP TABLE hbase_table_4
+
+DROP TABLE hbase_table_5
+
+DROP TABLE hbase_table_6
+
+DROP TABLE hbase_table_7
+
+DROP TABLE hbase_table_8
+
+DROP TABLE empty_hbase_table
+
+DROP TABLE empty_normal_table

Added: incubator/bigtop/trunk/test/src/smokes/hive/src/test/resources/scripts/integration/hbase_stats/filter
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/test/src/smokes/hive/src/test/resources/scripts/integration/hbase_stats/filter?rev=1148501&view=auto
==============================================================================
--- incubator/bigtop/trunk/test/src/smokes/hive/src/test/resources/scripts/integration/hbase_stats/filter (added)
+++ incubator/bigtop/trunk/test/src/smokes/hive/src/test/resources/scripts/integration/hbase_stats/filter Tue Jul 19 19:44:48 2011
@@ -0,0 +1,4 @@
+sed -e 's#hdfs://[^/]*/#HDFS_URL/#' \
+    -e '/^CreateTime:/d'            \
+    -e '/transient_lastDdlTime/d'   \
+    -e 's#^Owner:.*$#Owner: USER#'

Added: incubator/bigtop/trunk/test/src/smokes/hive/src/test/resources/scripts/integration/hbase_stats/in
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/test/src/smokes/hive/src/test/resources/scripts/integration/hbase_stats/in?rev=1148501&view=auto
==============================================================================
--- incubator/bigtop/trunk/test/src/smokes/hive/src/test/resources/scripts/integration/hbase_stats/in (added)
+++ incubator/bigtop/trunk/test/src/smokes/hive/src/test/resources/scripts/integration/hbase_stats/in Tue Jul 19 19:44:48 2011
@@ -0,0 +1,21 @@
+set datanucleus.cache.collections=false;
+
+set hive.stats.dbclass=hbase;
+
+create table stats_src like src;
+insert overwrite table stats_src select * from src;
+analyze table stats_src compute statistics;
+desc formatted stats_src;
+
+create table hbase_part like srcpart;
+
+insert overwrite table hbase_part partition (ds='2010-04-08', hr = '11') select key, value from src;
+insert overwrite table hbase_part partition (ds='2010-04-08', hr = '12') select key, value from src;
+
+analyze table hbase_part partition(ds='2008-04-08', hr=11) compute statistics;
+analyze table hbase_part partition(ds='2008-04-08', hr=12) compute statistics;
+
+desc formatted hbase_part;
+desc formatted hbase_part partition (ds='2010-04-08', hr = '11');
+desc formatted hbase_part partition (ds='2010-04-08', hr = '12');
+

Added: incubator/bigtop/trunk/test/src/smokes/hive/src/test/resources/scripts/integration/hbase_stats/out
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/test/src/smokes/hive/src/test/resources/scripts/integration/hbase_stats/out?rev=1148501&view=auto
==============================================================================
--- incubator/bigtop/trunk/test/src/smokes/hive/src/test/resources/scripts/integration/hbase_stats/out (added)
+++ incubator/bigtop/trunk/test/src/smokes/hive/src/test/resources/scripts/integration/hbase_stats/out Tue Jul 19 19:44:48 2011
@@ -0,0 +1,156 @@
+set datanucleus.cache.collections=false
+set hive.stats.dbclass=hbase
+
+
+create table stats_src like src
+
+insert overwrite table stats_src select * from src
+Deleted HDFS_URL/user/hive/warehouse/stats_src
+
+analyze table stats_src compute statistics
+
+desc formatted stats_src
+# col_name            	data_type           	comment             
+	 	 
+key                 	string              	None                
+value               	string              	None                
+	 	 
+# Detailed Table Information	 	 
+Database:           	default             	 
+Owner:              	testuser1@MINOTAUR.CLOUDERA.COM	 
+CreateTime:         	Fri Mar 25 11:29:26 PDT 2011	 
+LastAccessTime:     	UNKNOWN             	 
+Protect Mode:       	None                	 
+Retention:          	0                   	 
+Location:           	hdfs://minotaur01.sf.cloudera.com:17020/user/hive/warehouse/stats_src	 
+Table Type:         	MANAGED_TABLE       	 
+Table Parameters:	 	 
+	transient_lastDdlTime	1301077789          
+	 	 
+# Storage Information	 	 
+SerDe Library:      	org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe	 
+InputFormat:        	org.apache.hadoop.mapred.TextInputFormat	 
+OutputFormat:       	org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat	 
+Compressed:         	No                  	 
+Num Buckets:        	-1                  	 
+Bucket Columns:     	[]                  	 
+Sort Columns:       	[]                  	 
+Storage Desc Params:	 	 
+	serialization.format	1                   
+
+
+create table hbase_part like srcpart
+
+
+insert overwrite table hbase_part partition (ds='2010-04-08', hr = '11') select key, value from src
+
+insert overwrite table hbase_part partition (ds='2010-04-08', hr = '12') select key, value from src
+
+
+analyze table hbase_part partition(ds='2008-04-08', hr=11) compute statistics
+
+analyze table hbase_part partition(ds='2008-04-08', hr=12) compute statistics
+
+
+desc formatted hbase_part
+# col_name            	data_type           	comment             
+	 	 
+key                 	string              	None                
+value               	string              	None                
+	 	 
+# Partition Information	 	 
+# col_name            	data_type           	comment             
+	 	 
+ds                  	string              	None                
+hr                  	string              	None                
+	 	 
+# Detailed Table Information	 	 
+Database:           	default             	 
+Owner:              	testuser1@MINOTAUR.CLOUDERA.COM	 
+CreateTime:         	Fri Mar 25 11:30:31 PDT 2011	 
+LastAccessTime:     	UNKNOWN             	 
+Protect Mode:       	None                	 
+Retention:          	0                   	 
+Location:           	hdfs://minotaur01.sf.cloudera.com:17020/user/hive/warehouse/hbase_part	 
+Table Type:         	MANAGED_TABLE       	 
+Table Parameters:	 	 
+	transient_lastDdlTime	1301077831          
+	 	 
+# Storage Information	 	 
+SerDe Library:      	org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe	 
+InputFormat:        	org.apache.hadoop.mapred.TextInputFormat	 
+OutputFormat:       	org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat	 
+Compressed:         	No                  	 
+Num Buckets:        	-1                  	 
+Bucket Columns:     	[]                  	 
+Sort Columns:       	[]                  	 
+Storage Desc Params:	 	 
+	serialization.format	1                   
+
+desc formatted hbase_part partition (ds='2010-04-08', hr = '11')
+# col_name            	data_type           	comment             
+	 	 
+key                 	string              	None                
+value               	string              	None                
+	 	 
+# Partition Information	 	 
+# col_name            	data_type           	comment             
+	 	 
+ds                  	string              	None                
+hr                  	string              	None                
+	 	 
+# Detailed Partition Information	 	 
+Partition Value:    	[2010-04-08, 11]    	 
+Database:           	default             	 
+Table:              	hbase_part          	 
+CreateTime:         	Fri Mar 25 11:30:54 PDT 2011	 
+LastAccessTime:     	UNKNOWN             	 
+Protect Mode:       	None                	 
+Location:           	hdfs://minotaur01.sf.cloudera.com:17020/user/hive/warehouse/hbase_part/ds=2010-04-08/hr=11	 
+Partition Parameters:	 	 
+	transient_lastDdlTime	1301077854          
+	 	 
+# Storage Information	 	 
+SerDe Library:      	org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe	 
+InputFormat:        	org.apache.hadoop.mapred.TextInputFormat	 
+OutputFormat:       	org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat	 
+Compressed:         	No                  	 
+Num Buckets:        	-1                  	 
+Bucket Columns:     	[]                  	 
+Sort Columns:       	[]                  	 
+Storage Desc Params:	 	 
+	serialization.format	1                   
+
+desc formatted hbase_part partition (ds='2010-04-08', hr = '12')
+# col_name            	data_type           	comment             
+	 	 
+key                 	string              	None                
+value               	string              	None                
+	 	 
+# Partition Information	 	 
+# col_name            	data_type           	comment             
+	 	 
+ds                  	string              	None                
+hr                  	string              	None                
+	 	 
+# Detailed Partition Information	 	 
+Partition Value:    	[2010-04-08, 12]    	 
+Database:           	default             	 
+Table:              	hbase_part          	 
+CreateTime:         	Fri Mar 25 11:31:29 PDT 2011	 
+LastAccessTime:     	UNKNOWN             	 
+Protect Mode:       	None                	 
+Location:           	hdfs://minotaur01.sf.cloudera.com:17020/user/hive/warehouse/hbase_part/ds=2010-04-08/hr=12	 
+Partition Parameters:	 	 
+	transient_lastDdlTime	1301077889          
+	 	 
+# Storage Information	 	 
+SerDe Library:      	org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe	 
+InputFormat:        	org.apache.hadoop.mapred.TextInputFormat	 
+OutputFormat:       	org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat	 
+Compressed:         	No                  	 
+Num Buckets:        	-1                  	 
+Bucket Columns:     	[]                  	 
+Sort Columns:       	[]                  	 
+Storage Desc Params:	 	 
+	serialization.format	1                   

Added: incubator/bigtop/trunk/test/src/smokes/hive/src/test/resources/scripts/ql/authorization_2/filter
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/test/src/smokes/hive/src/test/resources/scripts/ql/authorization_2/filter?rev=1148501&view=auto
==============================================================================
--- incubator/bigtop/trunk/test/src/smokes/hive/src/test/resources/scripts/ql/authorization_2/filter (added)
+++ incubator/bigtop/trunk/test/src/smokes/hive/src/test/resources/scripts/ql/authorization_2/filter Tue Jul 19 19:44:48 2011
@@ -0,0 +1,2 @@
+sed -e 's#^grantTime.[0-9]*#grantTime\tJUSTNOW#' \
+    -e 's#^grantor.*$#grantor\t\tBORG#'



Mime
View raw message