incubator-hcatalog-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1235120 - in /incubator/hcatalog/branches/branch-0.3: ./ storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/ storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/ storage-drivers/hbase/src/test/org/apache/hcatalog/hbase/
Date Tue, 24 Jan 2012 03:32:44 GMT
Author: hashutosh
Date: Tue Jan 24 03:32:43 2012
New Revision: 1235120

URL: http://svn.apache.org/viewvc?rev=1235120&view=rev
Log:
HCATALOG-227 :  NPE on HBaseInputStorageDriver when table contains a CF that's never been
successfully written to (toffer via hashutosh)

Modified:
    incubator/hcatalog/branches/branch-0.3/CHANGES.txt
    incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HBaseInputStorageDriver.java
    incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HCatTableSnapshot.java
    incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/TableSnapshot.java
    incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/ZKBasedRevisionManager.java
    incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/src/test/org/apache/hcatalog/hbase/TestSnapshots.java

Modified: incubator/hcatalog/branches/branch-0.3/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.3/CHANGES.txt?rev=1235120&r1=1235119&r2=1235120&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.3/CHANGES.txt (original)
+++ incubator/hcatalog/branches/branch-0.3/CHANGES.txt Tue Jan 24 03:32:43 2012
@@ -105,6 +105,8 @@ Release 0.3.0 (unreleased changes)
   OPTIMIZATIONS
 
   BUG FIXES
+  HCAT-227. NPE on HBaseInputStorageDriver when table contains a CF that's never been successfully
written to (toffer via hashutosh)
+
   HCAT-226. HBase storage drivers should load hbase resources (hbase-site, hbase-default)
for the user (toffer via hashutosh)
 
   HCAT-224. hcatalog e2e tests have the hive version hard coded we should read that from
the hive build.properties (Arpit Gupta via hashutosh)

Modified: incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HBaseInputStorageDriver.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HBaseInputStorageDriver.java?rev=1235120&r1=1235119&r2=1235120&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HBaseInputStorageDriver.java
(original)
+++ incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HBaseInputStorageDriver.java
Tue Jan 24 03:32:43 2012
@@ -221,7 +221,7 @@ public class HBaseInputStorageDriver ext
         }
 
         HCatTableSnapshot hcatSnapshot = new HCatTableSnapshot(
-                 hcatTableInfo.getDatabaseName(), hcatTableInfo.getTableName(),revisionMap);
+                 hcatTableInfo.getDatabaseName(), hcatTableInfo.getTableName(),revisionMap,hbaseSnapshot.getLatestRevision());
         return hcatSnapshot;
     }
 
@@ -241,7 +241,7 @@ public class HBaseInputStorageDriver ext
 
         String fullyQualifiedName = hcatSnapshot.getDatabaseName() + "."
                 + hcatSnapshot.getTableName();
-        return new TableSnapshot(fullyQualifiedName, revisionMap);
+        return new TableSnapshot(fullyQualifiedName, revisionMap,hcatSnapshot.getLatestRevision());
 
     }
 

Modified: incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HCatTableSnapshot.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HCatTableSnapshot.java?rev=1235120&r1=1235119&r2=1235120&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HCatTableSnapshot.java
(original)
+++ incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HCatTableSnapshot.java
Tue Jan 24 03:32:43 2012
@@ -33,11 +33,13 @@ public class HCatTableSnapshot implement
     private String tableName;
     private String databaseName;
     private Map<String, Long> columnMap;
+    private long latestRevision;
 
-    HCatTableSnapshot(String databaseName, String tableName, Map<String, Long> columnMap)
{
+    HCatTableSnapshot(String databaseName, String tableName, Map<String, Long> columnMap,
long latestRevision) {
         this.tableName = tableName;
         this.databaseName = databaseName;
         this.columnMap = columnMap;
+        this.latestRevision = latestRevision;
     }
 
     /**
@@ -58,7 +60,9 @@ public class HCatTableSnapshot implement
      * @return The revision number of a column in a snapshot.
      */
     long getRevision(String column){
-        return this.columnMap.get(column);
+        if(columnMap.containsKey(column))
+            return this.columnMap.get(column);
+        return latestRevision;
     }
 
     /**
@@ -71,10 +75,17 @@ public class HCatTableSnapshot implement
         return this.columnMap.containsKey(column);
     }
 
+    /**
+     * @return latest committed revision when snapshot was taken
+     */
+    long getLatestRevision() {
+        return latestRevision;
+    }
+
     @Override
     public String toString() {
         String snapshot = " Database Name: " + this.databaseName +" Table Name : " + tableName
+
-                 " Column revision : " + columnMap.toString();
+                 "Latest Revision: "+latestRevision+" Column revision : " + columnMap.toString();
         return snapshot;
     }
 }

Modified: incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/TableSnapshot.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/TableSnapshot.java?rev=1235120&r1=1235119&r2=1235120&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/TableSnapshot.java
(original)
+++ incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/TableSnapshot.java
Tue Jan 24 03:32:43 2012
@@ -30,10 +30,13 @@ public class TableSnapshot {
 
     private Map<String, Long> cfRevisionMap;
 
+    private long latestRevision;
 
-    public TableSnapshot(String name, Map<String, Long> cfRevMap) {
+
+    public TableSnapshot(String name, Map<String, Long> cfRevMap, long latestRevision)
{
         this.name = name;
         this.cfRevisionMap = cfRevMap;
+        this.latestRevision = latestRevision;
     }
 
     /**
@@ -61,12 +64,21 @@ public class TableSnapshot {
      * @return the revision
      */
     public long getRevision(String familyName){
-        return this.cfRevisionMap.get(familyName);
+        if(cfRevisionMap.containsKey(familyName))
+            return cfRevisionMap.get(familyName);
+        return latestRevision;
+    }
+
+    /**
+     * @return the latest committed revision when this snapshot was taken
+     */
+    public long getLatestRevision() {
+        return latestRevision;
     }
 
     @Override
     public String toString() {
-        String snapshot = "Table Name : " + name
+        String snapshot = "Table Name : " + name +" Latest Revision: " + latestRevision
                 + " Column Familiy revision : " + cfRevisionMap.toString();
         return snapshot;
     }

Modified: incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/ZKBasedRevisionManager.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/ZKBasedRevisionManager.java?rev=1235120&r1=1235119&r2=1235120&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/ZKBasedRevisionManager.java
(original)
+++ incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/ZKBasedRevisionManager.java
Tue Jan 24 03:32:43 2012
@@ -320,7 +320,9 @@ public class ZKBasedRevisionManager impl
             cfMap.put(cfName, version);
         }
 
-        return new TableSnapshot(tableName, cfMap);
+        TableSnapshot snapshot = new TableSnapshot(tableName, cfMap,latestID);
+        LOG.debug("Created snapshot For table: "+tableName+" snapshot: "+snapshot);
+        return snapshot;
     }
 
     /* This method allows the user to create snapshot of a
@@ -346,7 +348,7 @@ public class ZKBasedRevisionManager impl
             cfMap.put(cf, revision);
         }
 
-        return new TableSnapshot(tableName, cfMap);
+        return new TableSnapshot(tableName, cfMap, revision);
     }
 
     /**

Modified: incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/src/test/org/apache/hcatalog/hbase/TestSnapshots.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/src/test/org/apache/hcatalog/hbase/TestSnapshots.java?rev=1235120&r1=1235119&r2=1235120&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/src/test/org/apache/hcatalog/hbase/TestSnapshots.java
(original)
+++ incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/src/test/org/apache/hcatalog/hbase/TestSnapshots.java
Tue Jan 24 03:32:43 2012
@@ -81,7 +81,7 @@ public class TestSnapshots extends Skele
         Map<String, Long> revMap = new HashMap<String, Long>();
         revMap.put("cf1", 3L);
         revMap.put("cf2", 5L);
-        TableSnapshot hbaseSnapshot = new TableSnapshot(fullyQualTableName, revMap);
+        TableSnapshot hbaseSnapshot = new TableSnapshot(fullyQualTableName, revMap,-1);
         HCatTableSnapshot hcatSnapshot = HBaseInputStorageDriver.convertSnapshot(hbaseSnapshot,
inputInfo.getTableInfo());
 
         assertEquals(hcatSnapshot.getRevision("value1"), 3);
@@ -101,7 +101,7 @@ public class TestSnapshots extends Skele
         assertEquals(0, cmdResponse.getResponseCode());
         revMap.clear();
         revMap.put("cf1", 3L);
-        hbaseSnapshot = new TableSnapshot(fullyQualTableName, revMap);
+        hbaseSnapshot = new TableSnapshot(fullyQualTableName, revMap, -1);
         inputInfo = InputJobInfo.create(databaseName, tableName, null, null, null);
         InitializeInput.setInput(job, inputInfo);
         modifiedInputInfo = job.getConfiguration().get(HCatConstants.HCAT_KEY_JOB_INFO);



Mime
View raw message