incubator-hcatalog-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tof...@apache.org
Subject svn commit: r1307031 - in /incubator/hcatalog/branches/branch-0.4: ./ storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/ storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/ storage-handlers/hbase/src/test/org/apache/hcatalog/hba...
Date Thu, 29 Mar 2012 18:25:09 GMT
Author: toffer
Date: Thu Mar 29 18:25:08 2012
New Revision: 1307031

URL: http://svn.apache.org/viewvc?rev=1307031&view=rev
Log:
merged from trunk: HCATALOG-301 Reading from a table created with upper case table name throws
error. (rohini via toffer)

Modified:
    incubator/hcatalog/branches/branch-0.4/   (props changed)
    incubator/hcatalog/branches/branch-0.4/CHANGES.txt
    incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseHCatStorageHandler.java
    incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManager.java
    incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpoint.java
    incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpointClient.java
    incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/ZKBasedRevisionManager.java
    incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseBulkOutputFormat.java
    incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseDirectOutputFormat.java
    incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseHCatStorageHandler.java
    incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseInputFormat.java
    incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManagerEndpoint.java

Propchange: incubator/hcatalog/branches/branch-0.4/
------------------------------------------------------------------------------
  Merged /incubator/hcatalog/trunk:r1307029

Modified: incubator/hcatalog/branches/branch-0.4/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/CHANGES.txt?rev=1307031&r1=1307030&r2=1307031&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/CHANGES.txt (original)
+++ incubator/hcatalog/branches/branch-0.4/CHANGES.txt Thu Mar 29 18:25:08 2012
@@ -90,6 +90,8 @@ Release 0.4.0 - Unreleased
   OPTIMIZATIONS
 
   BUG FIXES
+  HCAT-301 Reading from a table created with upper case table name throws error. (rohini
via toffer)
+
   HCAT-319 Cleanup of 0.3 mapred classes (khorgath via gates)
 
   HCAT-302 unable to write to hbase channel. HBaseHCatStorageHandler class not found (rohini
via gates)

Modified: incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseHCatStorageHandler.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseHCatStorageHandler.java?rev=1307031&r1=1307030&r2=1307031&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseHCatStorageHandler.java
(original)
+++ incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseHCatStorageHandler.java
Thu Mar 29 18:25:08 2012
@@ -60,7 +60,6 @@ import org.apache.hcatalog.hbase.HBaseBu
 import org.apache.hcatalog.hbase.HBaseDirectOutputFormat.HBaseDirectOutputCommitter;
 import org.apache.hcatalog.hbase.snapshot.RevisionManager;
 import org.apache.hcatalog.hbase.snapshot.Transaction;
-import org.apache.hcatalog.hbase.snapshot.ZKBasedRevisionManager;
 import org.apache.hcatalog.mapreduce.HCatOutputFormat;
 import org.apache.hcatalog.mapreduce.HCatTableInfo;
 import org.apache.hcatalog.mapreduce.InputJobInfo;
@@ -83,6 +82,7 @@ public class HBaseHCatStorageHandler ext
     private final static String PROPERTY_INT_OUTPUT_LOCATION = "hcat.hbase.mapreduce.intermediateOutputLocation";
 
     private Configuration hbaseConf;
+    private Configuration jobConf;
     private HBaseAdmin admin;
 
     @Override
@@ -94,17 +94,18 @@ public class HBaseHCatStorageHandler ext
         try {
             InputJobInfo inputJobInfo = (InputJobInfo) HCatUtil.deserialize(jobString);
             HCatTableInfo tableInfo = inputJobInfo.getTableInfo();
-            String qualifiedTableName = HBaseHCatStorageHandler.getFullyQualifiedName(tableInfo);
+            String qualifiedTableName = HBaseHCatStorageHandler.getFullyQualifiedHBaseTableName(tableInfo);
             jobProperties.put(TableInputFormat.INPUT_TABLE, qualifiedTableName);
 
-            Configuration jobConf = getConf();
+            Configuration jobConf = getJobConf();
             addHbaseResources(jobConf, jobProperties);
-            Configuration copyOfConf = new Configuration(jobConf);
+            JobConf copyOfConf = new JobConf(jobConf);
             HBaseConfiguration.addHbaseResources(copyOfConf);
             //Getting hbase delegation token in getInputSplits does not work with PIG. So
need to
             //do it here
-            if (jobConf instanceof JobConf) {
-                HBaseUtil.addHBaseDelegationToken((JobConf)jobConf);
+            if (jobConf instanceof JobConf) { //Should be the case
+                HBaseUtil.addHBaseDelegationToken(copyOfConf);
+                ((JobConf)jobConf).getCredentials().addAll(copyOfConf.getCredentials());
             }
 
             String outputSchema = jobConf.get(HCatConstants.HCAT_KEY_OUTPUT_SCHEMA);
@@ -140,11 +141,11 @@ public class HBaseHCatStorageHandler ext
         try {
             OutputJobInfo outputJobInfo = (OutputJobInfo) HCatUtil.deserialize(jobString);
             HCatTableInfo tableInfo = outputJobInfo.getTableInfo();
-            String qualifiedTableName = HBaseHCatStorageHandler.getFullyQualifiedName(tableInfo);
+            String qualifiedTableName = HBaseHCatStorageHandler.getFullyQualifiedHBaseTableName(tableInfo);
             jobProperties.put(HBaseConstants.PROPERTY_OUTPUT_TABLE_NAME_KEY, qualifiedTableName);
             jobProperties.put(TableOutputFormat.OUTPUT_TABLE, qualifiedTableName);
 
-            Configuration jobConf = getConf();
+            Configuration jobConf = getJobConf();
             addHbaseResources(jobConf, jobProperties);
 
             Configuration copyOfConf = new Configuration(jobConf);
@@ -152,24 +153,14 @@ public class HBaseHCatStorageHandler ext
 
             String txnString = outputJobInfo.getProperties().getProperty(
                     HBaseConstants.PROPERTY_WRITE_TXN_KEY);
-            String jobTxnString = jobConf.get(HBaseConstants.PROPERTY_WRITE_TXN_KEY);
-            //Pig makes 3 calls to HCatOutputFormat.setOutput(HCatStorer) with different
JobConf
-            //which leads to creating 2 transactions.
-            //So apart from fixing HCatStorer to pass same OutputJobInfo, making the call
idempotent for other
-            //cases which might call multiple times but with same JobConf.
             Transaction txn = null;
-            if (txnString == null && jobTxnString == null) {
+            if (txnString == null) {
                 txn = HBaseRevisionManagerUtil.beginWriteTransaction(qualifiedTableName,
tableInfo, copyOfConf);
                 String serializedTxn = HCatUtil.serialize(txn);
                 outputJobInfo.getProperties().setProperty(HBaseConstants.PROPERTY_WRITE_TXN_KEY,
                         serializedTxn);
-                jobProperties.put(HBaseConstants.PROPERTY_WRITE_TXN_KEY, serializedTxn);
             } else {
-                txnString = (txnString == null) ? jobTxnString : txnString;
                 txn = (Transaction) HCatUtil.deserialize(txnString);
-                outputJobInfo.getProperties().setProperty(HBaseConstants.PROPERTY_WRITE_TXN_KEY,
-                        txnString);
-                jobProperties.put(HBaseConstants.PROPERTY_WRITE_TXN_KEY, txnString);
             }
             if (isBulkMode(outputJobInfo)) {
                 String tableLocation = tableInfo.getTableLocation();
@@ -259,7 +250,7 @@ public class HBaseHCatStorageHandler ext
         }
 
         try {
-            String tableName = getHBaseTableName(tbl);
+            String tableName = getFullyQualifiedHBaseTableName(tbl);
             String hbaseColumnsMapping = tbl.getParameters().get(
                     HBaseSerDe.HBASE_COLUMNS_MAPPING);
 
@@ -331,13 +322,9 @@ public class HBaseHCatStorageHandler ext
             // ensure the table is online
             new HTable(hbaseConf, tableDesc.getName());
 
-            //Set up znodes in revision manager.
+            //Set up table in revision manager.
             RevisionManager rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(hbaseConf);
-            if (rm instanceof ZKBasedRevisionManager) {
-                ZKBasedRevisionManager zkRM = (ZKBasedRevisionManager) rm;
-                zkRM.setUpZNodes(tableName, new ArrayList<String>(
-                        uniqueColumnFamilies));
-            }
+            rm.createTable(tableName, new ArrayList<String>(uniqueColumnFamilies));
 
         } catch (MasterNotRunningException mnre) {
             throw new MetaException(StringUtils.stringifyException(mnre));
@@ -409,7 +396,7 @@ public class HBaseHCatStorageHandler ext
         }
     }
 
-    private String getHBaseTableName(Table tbl) {
+    private String getFullyQualifiedHBaseTableName(Table tbl) {
         String tableName = tbl.getParameters().get(HBaseSerDe.HBASE_TABLE_NAME);
         if (tableName == null) {
             tableName = tbl.getSd().getSerdeInfo().getParameters()
@@ -421,10 +408,28 @@ public class HBaseHCatStorageHandler ext
             } else {
                 tableName = tbl.getDbName() + "." + tbl.getTableName();
             }
+            tableName = tableName.toLowerCase();
         }
         return tableName;
     }
 
+    static String getFullyQualifiedHBaseTableName(HCatTableInfo tableInfo){
+        String qualifiedName = tableInfo.getStorerInfo().getProperties()
+                .getProperty(HBaseSerDe.HBASE_TABLE_NAME);
+        if (qualifiedName == null) {
+            String databaseName = tableInfo.getDatabaseName();
+            String tableName = tableInfo.getTableName();
+            if ((databaseName == null)
+                    || (databaseName.equals(MetaStoreUtils.DEFAULT_DATABASE_NAME))) {
+                qualifiedName = tableName;
+            } else {
+                qualifiedName = databaseName + "." + tableName;
+            }
+            qualifiedName = qualifiedName.toLowerCase();
+        }
+        return qualifiedName;
+    }
+
     @Override
     public Class<? extends InputFormat> getInputFormatClass() {
         return HBaseInputFormat.class;
@@ -449,6 +454,10 @@ public class HBaseHCatStorageHandler ext
         return HBaseSerDe.class;
     }
 
+    public Configuration getJobConf() {
+        return jobConf;
+    }
+
     @Override
     public Configuration getConf() {
 
@@ -460,15 +469,23 @@ public class HBaseHCatStorageHandler ext
 
     @Override
     public void setConf(Configuration conf) {
-        //Not cloning as we want to set tmpjars on it. Putting in jobProperties does not
-        //get propagated to JobConf in case of InputFormat as they are maintained per partition.
-        //Also we need to add hbase delegation token to the Credentials.
-        hbaseConf = conf;
+        //setConf is called both during DDL operations and  mapred read/write jobs.
+        //Creating a copy of conf for DDL and adding hbase-default and hbase-site.xml to
it.
+        //For jobs, maintaining a reference instead of cloning as we need to
+        //  1) add hbase delegation token to the Credentials.
+        //  2) set tmpjars on it. Putting in jobProperties does not get propagated to JobConf
+        //     in case of InputFormat as they are maintained per partition.
+        //Not adding hbase-default.xml and hbase-site.xml to jobConf as it will override
any
+        //hbase properties set in the JobConf by the user. In configureInputJobProperties
and
+        //configureOutputJobProperties, we take care of adding the default properties
+        //that are not already present. TODO: Change to a copy for jobs after HCAT-308 is
fixed.
+        jobConf = conf;
+        hbaseConf = HBaseConfiguration.create(conf);
     }
 
     private void checkDeleteTable(Table table) throws MetaException {
         boolean isExternal = MetaStoreUtils.isExternalTable(table);
-        String tableName = getHBaseTableName(table);
+        String tableName = getFullyQualifiedHBaseTableName(table);
         RevisionManager rm = null;
         try {
             if (!isExternal && getHBaseAdmin().tableExists(tableName)) {
@@ -478,12 +495,9 @@ public class HBaseHCatStorageHandler ext
                 }
                 getHBaseAdmin().deleteTable(tableName);
 
-              //Set up znodes in revision manager.
+                //Drop table in revision manager.
                 rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(hbaseConf);
-                if (rm instanceof ZKBasedRevisionManager) {
-                    ZKBasedRevisionManager zkRM = (ZKBasedRevisionManager) rm;
-                    zkRM.deleteZNodes(tableName);
-                }
+                rm.dropTable(tableName);
             }
         } catch (IOException ie) {
             throw new MetaException(StringUtils.stringifyException(ie));
@@ -492,20 +506,6 @@ public class HBaseHCatStorageHandler ext
         }
     }
 
-    static String getFullyQualifiedName(HCatTableInfo tableInfo){
-        String qualifiedName;
-        String databaseName = tableInfo.getDatabaseName();
-        String tableName = tableInfo.getTableName();
-
-        if ((databaseName == null) || (databaseName.equals(MetaStoreUtils.DEFAULT_DATABASE_NAME)))
{
-            qualifiedName = tableName;
-        } else {
-            qualifiedName = databaseName + "." + tableName;
-        }
-
-        return qualifiedName;
-    }
-
     /**
      * Helper method for users to add the required depedency jars to distributed cache.
      * @param conf

Modified: incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManager.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManager.java?rev=1307031&r1=1307030&r2=1307031&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManager.java
(original)
+++ incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManager.java
Thu Mar 29 18:25:08 2012
@@ -51,6 +51,19 @@ public interface RevisionManager {
     public void close() throws IOException;
 
     /**
+     * Setup revision management for a newly created hbase table.
+     * @param table the hbase table name
+     * @param columnFamilies the column families in the table
+     */
+    public void createTable(String table, List<String> columnFamilies) throws IOException;
+
+    /**
+     * Remove table data from revision manager for a dropped table.
+     * @param table the hbase table name
+     */
+    public void dropTable(String table) throws IOException;
+
+    /**
      * Start the write transaction.
      *
      * @param table

Modified: incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpoint.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpoint.java?rev=1307031&r1=1307030&r2=1307031&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpoint.java
(original)
+++ incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpoint.java
Thu Mar 29 18:25:08 2012
@@ -83,6 +83,16 @@ public class RevisionManagerEndpoint ext
   }
 
   @Override
+  public void createTable(String table, List<String> columnFamilies) throws IOException
{
+    rmImpl.createTable(table, columnFamilies);
+  }
+
+  @Override
+  public void dropTable(String table) throws IOException {
+    rmImpl.dropTable(table);
+  }
+
+  @Override
   public Transaction beginWriteTransaction(String table, List<String> families)
       throws IOException {
     return rmImpl.beginWriteTransaction(table, families);

Modified: incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpointClient.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpointClient.java?rev=1307031&r1=1307030&r2=1307031&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpointClient.java
(original)
+++ incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpointClient.java
Thu Mar 29 18:25:08 2012
@@ -53,6 +53,16 @@ public class RevisionManagerEndpointClie
   }
 
   @Override
+  public void createTable(String table, List<String> columnFamilies) throws IOException
{
+    rmProxy.createTable(table, columnFamilies);
+  }
+
+  @Override
+  public void dropTable(String table) throws IOException {
+    rmProxy.dropTable(table);
+  }
+
+  @Override
   public Transaction beginWriteTransaction(String table, List<String> families) throws
IOException {
     return rmProxy.beginWriteTransaction(table, families);
   }

Modified: incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/ZKBasedRevisionManager.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/ZKBasedRevisionManager.java?rev=1307031&r1=1307030&r2=1307031&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/ZKBasedRevisionManager.java
(original)
+++ incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/ZKBasedRevisionManager.java
Thu Mar 29 18:25:08 2012
@@ -88,6 +88,16 @@ public class ZKBasedRevisionManager impl
         }
     }
 
+    @Override
+    public void createTable(String table, List<String> columnFamilies) throws IOException
{
+        zkUtil.createRootZNodes();
+        zkUtil.setUpZnodesForTable(table, columnFamilies);
+    }
+
+    @Override
+    public void dropTable(String table) throws IOException {
+        zkUtil.deleteZNodes(table);
+    }
 
     /* @param table
     /* @param families
@@ -413,29 +423,6 @@ public class ZKBasedRevisionManager impl
          return lockPath;
      }
 
-     /**
-      * Sets up the table, column family znodes in zookeeper.
-      *
-      * @param tableName the hbase table name
-      * @param columnFamilies the column families in hbase
-      * @throws IOException Signals that an I/O exception has occurred.
-      */
-     public void setUpZNodes(String tableName, List<String> columnFamilies) throws
IOException{
-         zkUtil.createRootZNodes();
-         zkUtil.setUpZnodesForTable(tableName, columnFamilies);
-     }
-
-     /**
-      * Delete the table znodes from zookeeper.
-      *
-      * @param tableName the table name
-      * @throws IOException Signals that an I/O exception has occurred.
-      */
-     public void deleteZNodes(String tableName) throws IOException {
-         zkUtil.deleteZNodes(tableName);
-     }
-
-
     /*
      * This class is a listener class for the locks used in revision management.
      * TBD: Use the following class to signal that that the lock is actually

Modified: incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseBulkOutputFormat.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseBulkOutputFormat.java?rev=1307031&r1=1307030&r2=1307031&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseBulkOutputFormat.java
(original)
+++ incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseBulkOutputFormat.java
Thu Mar 29 18:25:08 2012
@@ -552,7 +552,6 @@ public class TestHBaseBulkOutputFormat e
         Job job = configureJob(testName,
                 conf, workingDir, MapWriteAbortTransaction.class,
                 outputJobInfo, inputPath);
-        job.waitForCompletion(true);
         assertFalse(job.waitForCompletion(true));
 
         // verify that revision manager has it as aborted transaction

Modified: incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseDirectOutputFormat.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseDirectOutputFormat.java?rev=1307031&r1=1307030&r2=1307031&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseDirectOutputFormat.java
(original)
+++ incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseDirectOutputFormat.java
Thu Mar 29 18:25:08 2012
@@ -187,12 +187,13 @@ public class TestHBaseDirectOutputFormat
         String testName = "directHCatOutputFormatTest";
         Path methodTestDir = new Path(getTestDir(),testName);
 
-        String databaseName = testName.toLowerCase();
+        String databaseName = testName;
         String dbDir = new Path(methodTestDir,"DB_"+testName).toString();
-        String tableName = newTableName(testName).toLowerCase();
+        String tableName = newTableName(testName);
         String familyName = "my_family";
         byte[] familyNameBytes = Bytes.toBytes(familyName);
-
+        //Table name will be lower case unless specified by hbase.table.name property
+        String hbaseTableName = (databaseName + "." + tableName).toLowerCase();
 
         //include hbase config in conf file
         Configuration conf = new Configuration(allConf);
@@ -233,7 +234,7 @@ public class TestHBaseDirectOutputFormat
 
         RevisionManager rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(conf);
         try {
-            TableSnapshot snapshot = rm.createSnapshot(databaseName+"."+tableName);
+            TableSnapshot snapshot = rm.createSnapshot(hbaseTableName);
             for(String el: snapshot.getColumnFamilies()) {
                 assertEquals(1,snapshot.getRevision(el));
             }
@@ -242,7 +243,7 @@ public class TestHBaseDirectOutputFormat
         }
 
         //verify
-        HTable table = new HTable(conf, databaseName+"."+tableName);
+        HTable table = new HTable(conf, hbaseTableName);
         Scan scan = new Scan();
         scan.addFamily(familyNameBytes);
         ResultScanner scanner = table.getScanner(scan);
@@ -264,11 +265,13 @@ public class TestHBaseDirectOutputFormat
     public void directModeAbortTest() throws Exception {
         String testName = "directModeAbortTest";
         Path methodTestDir = new Path(getTestDir(), testName);
-        String databaseName = testName.toLowerCase();
+        String databaseName = testName;
         String dbDir = new Path(methodTestDir, "DB_" + testName).toString();
-        String tableName = newTableName(testName).toLowerCase();
+        String tableName = newTableName(testName);
         String familyName = "my_family";
         byte[] familyNameBytes = Bytes.toBytes(familyName);
+        //Table name as specified by hbase.table.name property
+        String hbaseTableName = tableName;
 
         // include hbase config in conf file
         Configuration conf = new Configuration(allConf);
@@ -281,7 +284,7 @@ public class TestHBaseDirectOutputFormat
                 "'org.apache.hcatalog.hbase.HBaseHCatStorageHandler'" +
                 "TBLPROPERTIES (" +
                 "'hbase.columns.mapping'=':key," + familyName + ":english," + familyName
+
-                ":spanish')";
+                ":spanish','hbase.table.name'='"+ hbaseTableName +"')";
 
         assertEquals(0, hcatDriver.run(dbquery).getResponseCode());
         assertEquals(0, hcatDriver.run(tableQuery).getResponseCode());
@@ -305,17 +308,16 @@ public class TestHBaseDirectOutputFormat
                 tableName, null);
         Job job = configureJob(testName, conf, workingDir, MapWriteAbortTransaction.class,
                 outputJobInfo, inputPath);
-        job.waitForCompletion(true);
         assertFalse(job.waitForCompletion(true));
 
         // verify that revision manager has it as aborted transaction
         RevisionManager rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(conf);
         try {
-            TableSnapshot snapshot = rm.createSnapshot(databaseName + "." + tableName);
+            TableSnapshot snapshot = rm.createSnapshot(hbaseTableName);
             for (String family : snapshot.getColumnFamilies()) {
                 assertEquals(1, snapshot.getRevision(family));
                 List<FamilyRevision> abortedWriteTransactions = rm.getAbortedWriteTransactions(
-                        databaseName + "." + tableName, family);
+                        hbaseTableName, family);
                 assertEquals(1, abortedWriteTransactions.size());
                 assertEquals(1, abortedWriteTransactions.get(0).getRevision());
             }
@@ -324,7 +326,7 @@ public class TestHBaseDirectOutputFormat
         }
 
         // verify that hbase has the records of the successful maps.
-        HTable table = new HTable(conf, databaseName + "." + tableName);
+        HTable table = new HTable(conf, hbaseTableName);
         Scan scan = new Scan();
         scan.addFamily(familyNameBytes);
         ResultScanner scanner = table.getScanner(scan);
@@ -380,12 +382,8 @@ public class TestHBaseDirectOutputFormat
         job.setOutputFormatClass(HCatOutputFormat.class);
         HCatOutputFormat.setOutput(job, outputJobInfo);
         String txnString = job.getConfiguration().get(HBaseConstants.PROPERTY_WRITE_TXN_KEY);
-        //Test passing in same jobConf or same OutputJobInfo multiple times and verify 1
transaction is created
-        //Same jobConf
-        HCatOutputFormat.setOutput(job, outputJobInfo);
-        assertEquals(txnString, job.getConfiguration().get(HBaseConstants.PROPERTY_WRITE_TXN_KEY));
+        //Test passing in same OutputJobInfo multiple times and verify 1 transaction is created
         String jobString = job.getConfiguration().get(HCatConstants.HCAT_KEY_OUTPUT_INFO);
-        //Same OutputJobInfo
         outputJobInfo = (OutputJobInfo) HCatUtil.deserialize(jobString);
         Job job2 = new Job(conf);
         HCatOutputFormat.setOutput(job2, outputJobInfo);

Modified: incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseHCatStorageHandler.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseHCatStorageHandler.java?rev=1307031&r1=1307030&r2=1307031&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseHCatStorageHandler.java
(original)
+++ incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseHCatStorageHandler.java
Thu Mar 29 18:25:08 2012
@@ -36,6 +36,8 @@ import org.apache.hadoop.hive.ql.process
 import org.apache.hadoop.hive.ql.session.SessionState;
 import org.apache.hcatalog.cli.HCatDriver;
 import org.apache.hcatalog.cli.SemanticAnalysis.HCatSemanticAnalyzer;
+import org.apache.hcatalog.hbase.snapshot.RevisionManager;
+import org.apache.zookeeper.KeeperException.NoNodeException;
 import org.junit.Test;
 
 public class TestHBaseHCatStorageHandler extends SkeletonHBaseTest {
@@ -85,11 +87,94 @@ public class TestHBaseHCatStorageHandler
 
         assertTrue(doesTableExist);
 
+        RevisionManager rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(getHbaseConf());
+        rm.open();
+        //Should be able to successfully query revision manager
+        rm.getAbortedWriteTransactions("test_table", "cf1");
+
         hcatDriver.run("drop table test_table");
         doesTableExist = hAdmin.tableExists("test_table");
+        assertTrue(doesTableExist == false);
+
+        try {
+            rm.getAbortedWriteTransactions("test_table", "cf1");
+        } catch (Exception e) {
+            assertTrue(e.getCause() instanceof NoNodeException);
+        }
+        rm.close();
+
+    }
+
+    @Test
+    public void testTableCreateDropDifferentCase() throws Exception {
+        Initialize();
+
+        hcatDriver.run("drop table test_Table");
+        CommandProcessorResponse response = hcatDriver
+                .run("create table test_Table(key int, value string) STORED BY " +
+                             "'org.apache.hcatalog.hbase.HBaseHCatStorageHandler'"
+                    + "TBLPROPERTIES ('hbase.columns.mapping'=':key,cf1:val')");
+
+        assertEquals(0, response.getResponseCode());
+
+        //HBase table gets created with lower case unless specified as a table property.
+        HBaseAdmin hAdmin = new HBaseAdmin(getHbaseConf());
+        boolean doesTableExist = hAdmin.tableExists("test_table");
 
+        assertTrue(doesTableExist);
+
+        RevisionManager rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(getHbaseConf());
+        rm.open();
+        //Should be able to successfully query revision manager
+        rm.getAbortedWriteTransactions("test_table", "cf1");
+
+        hcatDriver.run("drop table test_table");
+        doesTableExist = hAdmin.tableExists("test_table");
         assertTrue(doesTableExist == false);
 
+        try {
+            rm.getAbortedWriteTransactions("test_table", "cf1");
+        } catch (Exception e) {
+            assertTrue(e.getCause() instanceof NoNodeException);
+        }
+        rm.close();
+
+    }
+
+    @Test
+    public void testTableCreateDropCaseSensitive() throws Exception {
+        Initialize();
+
+        hcatDriver.run("drop table test_Table");
+        CommandProcessorResponse response = hcatDriver
+                .run("create table test_Table(key int, value string) STORED BY " +
+                             "'org.apache.hcatalog.hbase.HBaseHCatStorageHandler'"
+                    + "TBLPROPERTIES ('hbase.columns.mapping'=':key,cf1:val'," +
+                    " 'hbase.table.name'='CaseSensitiveTable')");
+
+        assertEquals(0, response.getResponseCode());
+
+        HBaseAdmin hAdmin = new HBaseAdmin(getHbaseConf());
+        boolean doesTableExist = hAdmin.tableExists("CaseSensitiveTable");
+
+        assertTrue(doesTableExist);
+
+        RevisionManager rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(getHbaseConf());
+        rm.open();
+        //Should be able to successfully query revision manager
+        rm.getAbortedWriteTransactions("CaseSensitiveTable", "cf1");
+
+        hcatDriver.run("drop table test_table");
+        doesTableExist = hAdmin.tableExists("CaseSensitiveTable");
+        assertTrue(doesTableExist == false);
+
+        try {
+            rm.getAbortedWriteTransactions("CaseSensitiveTable", "cf1");
+        } catch (Exception e) {
+            assertTrue(e.getCause() instanceof NoNodeException);
+        }
+        rm.close();
+
     }
 
     @Test

Modified: incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseInputFormat.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseInputFormat.java?rev=1307031&r1=1307030&r2=1307031&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseInputFormat.java
(original)
+++ incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseInputFormat.java
Thu Mar 29 18:25:08 2012
@@ -172,8 +172,10 @@ public class TestHBaseInputFormat extend
     @Test
     public void TestHBaseTableReadMR() throws Exception {
         Initialize();
-        String tableName = newTableName("mytable");
-        String databaseName = newTableName("mydatabase");
+        String tableName = newTableName("MyTable");
+        String databaseName = newTableName("MyDatabase");
+        //Table name will be lower case unless specified by hbase.table.name property
+        String hbaseTableName = (databaseName + "." + tableName).toLowerCase();
         String db_dir = getTestDir() + "/hbasedb";
 
         String dbquery = "CREATE DATABASE IF NOT EXISTS " + databaseName + " LOCATION '"
@@ -189,7 +191,6 @@ public class TestHBaseInputFormat extend
         assertEquals(0, responseTwo.getResponseCode());
 
         HBaseAdmin hAdmin = new HBaseAdmin(getHbaseConf());
-        String hbaseTableName = databaseName + "." + tableName;
         boolean doesTableExist = hAdmin.tableExists(hbaseTableName);
         assertTrue(doesTableExist);
 
@@ -243,21 +244,24 @@ public class TestHBaseInputFormat extend
     public void TestHBaseTableProjectionReadMR() throws Exception {
 
         Initialize();
-        String tableName = newTableName("mytable");
+        String tableName = newTableName("MyTable");
+        //Table name as specified by hbase.table.name property
+        String hbaseTableName = "MyDB_" + tableName;
         String tableQuery = "CREATE TABLE " + tableName
-                              + "(key string, testqualifier1 string, testqualifier2 string)
STORED BY " +
-                              "'org.apache.hcatalog.hbase.HBaseHCatStorageHandler'"
-                              + "TBLPROPERTIES ('hbase.columns.mapping'=':key," +
-                              		"testFamily:testQualifier1,testFamily:testQualifier2')" ;
+                              + "(key string, testqualifier1 string, testqualifier2 string)
STORED BY "
+                              + "'org.apache.hcatalog.hbase.HBaseHCatStorageHandler'"
+                              + "TBLPROPERTIES ('hbase.columns.mapping'="
+                              + "':key,testFamily:testQualifier1,testFamily:testQualifier2',"
+                              + "'hbase.table.name'='" + hbaseTableName+ "')" ;
 
         CommandProcessorResponse responseTwo = hcatDriver.run(tableQuery);
         assertEquals(0, responseTwo.getResponseCode());
 
         HBaseAdmin hAdmin = new HBaseAdmin(getHbaseConf());
-        boolean doesTableExist = hAdmin.tableExists(tableName);
+        boolean doesTableExist = hAdmin.tableExists(hbaseTableName);
         assertTrue(doesTableExist);
 
-        populateHBaseTable(tableName, 5);
+        populateHBaseTable(hbaseTableName, 5);
 
         Configuration conf = new Configuration(hcatConf);
         conf.set(HCatConstants.HCAT_KEY_HIVE_CONF,
@@ -293,7 +297,7 @@ public class TestHBaseInputFormat extend
         CommandProcessorResponse responseThree = hcatDriver.run(dropTableQuery);
         assertEquals(0, responseThree.getResponseCode());
 
-        boolean isHbaseTableThere = hAdmin.tableExists(tableName);
+        boolean isHbaseTableThere = hAdmin.tableExists(hbaseTableName);
         assertFalse(isHbaseTableThere);
     }
 

Modified: incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManagerEndpoint.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManagerEndpoint.java?rev=1307031&r1=1307030&r2=1307031&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManagerEndpoint.java
(original)
+++ incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManagerEndpoint.java
Thu Mar 29 18:25:08 2012
@@ -121,6 +121,14 @@ public class TestRevisionManagerEndpoint
     }
 
     @Override
+    public void createTable(String table, List<String> columnFamilies) throws IOException
{
+    }
+
+    @Override
+    public void dropTable(String table) throws IOException {
+    }
+
+    @Override
     public Transaction beginWriteTransaction(String table,
         List<String> families) throws IOException {
       return recordCall(null, table, families);



Mime
View raw message