gora-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From fe...@apache.org
Subject svn commit: r1295608 - in /gora/trunk: ./ gora-hbase/src/main/java/org/apache/gora/hbase/store/
Date Thu, 01 Mar 2012 15:08:21 GMT
Author: ferdy
Date: Thu Mar  1 15:08:20 2012
New Revision: 1295608

URL: http://svn.apache.org/viewvc?rev=1295608&view=rev
Log:
GORA-101 HBaseStore should properly support multiple tables in the mapping file.

Modified:
    gora/trunk/CHANGES.txt
    gora/trunk/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseColumn.java
    gora/trunk/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseMapping.java
    gora/trunk/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseStore.java

Modified: gora/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/gora/trunk/CHANGES.txt?rev=1295608&r1=1295607&r2=1295608&view=diff
==============================================================================
--- gora/trunk/CHANGES.txt (original)
+++ gora/trunk/CHANGES.txt Thu Mar  1 15:08:20 2012
@@ -6,6 +6,8 @@ Gora Change Log
 
 Trunk (unreleased changes):
 
+* GORA-101 HBaseStore should properly support multiple tables in the mapping file. (ferdy)
+
 * GORA-82 Add missing license headers & RAT target to pom.xml (lewismc)
 
 * GORA-88 HBaseByteInterface not thread safe (ferdy)

Modified: gora/trunk/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseColumn.java
URL: http://svn.apache.org/viewvc/gora/trunk/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseColumn.java?rev=1295608&r1=1295607&r2=1295608&view=diff
==============================================================================
--- gora/trunk/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseColumn.java (original)
+++ gora/trunk/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseColumn.java Thu Mar
 1 15:08:20 2012
@@ -20,24 +20,18 @@ package org.apache.gora.hbase.store;
 import java.util.Arrays;
 
 /**
- * Stores tablename, family, qualifier tuple 
+ * Store family, qualifier tuple 
  */
 class HBaseColumn {
   
-  final String tableName;
   final byte[] family;
   final byte[] qualifier;
   
-  public HBaseColumn(String tableName, byte[] family, byte[] qualifier) {
-    this.tableName = tableName;
+  public HBaseColumn(byte[] family, byte[] qualifier) {
     this.family = family==null ? null : Arrays.copyOf(family, family.length);
     this.qualifier = qualifier==null ? null : 
       Arrays.copyOf(qualifier, qualifier.length);
   }
-
-  public String getTableName() {
-    return tableName;
-  }
   
   /**
    * @return the family (internal array returned; do not modify)
@@ -77,4 +71,12 @@ class HBaseColumn {
       return false;
     return true;
   }
+
+  @Override
+  public String toString() {
+    return "HBaseColumn [family=" + Arrays.toString(family) + ", qualifier="
+        + Arrays.toString(qualifier) + "]";
+  }
+  
+  
 }

Modified: gora/trunk/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseMapping.java
URL: http://svn.apache.org/viewvc/gora/trunk/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseMapping.java?rev=1295608&r1=1295607&r2=1295608&view=diff
==============================================================================
--- gora/trunk/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseMapping.java (original)
+++ gora/trunk/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseMapping.java Thu
Mar  1 15:08:20 2012
@@ -29,35 +29,29 @@ import org.apache.hadoop.hbase.util.Byte
 
 /**
  * Mapping definitions for HBase. Thread safe.
+ * It holds a definition for a single table. 
  */
 public class HBaseMapping {
 
-  private final Map<String, HTableDescriptor> tableDescriptors;
-  
-  //name of the primary table
-  private final String tableName; 
+  private final HTableDescriptor tableDescriptor;
   
   // a map from field name to hbase column
   private final Map<String, HBaseColumn> columnMap;
   
-  public HBaseMapping(Map<String, HTableDescriptor> tableDescriptors,
-      String tableName, Map<String, HBaseColumn> columnMap) {
+  public HBaseMapping(HTableDescriptor tableDescriptor,
+      Map<String, HBaseColumn> columnMap) {
     super();
-    this.tableDescriptors = tableDescriptors;
-    this.tableName = tableName;
+    this.tableDescriptor = tableDescriptor;
     this.columnMap = columnMap;
   }
+  
 
   public String getTableName() {
-    return tableName;
+    return tableDescriptor.getNameAsString();
   }
   
   public HTableDescriptor getTable() {
-    return getTable(tableName);
-  }
-  
-  public HTableDescriptor getTable(String tableName) {
-    return tableDescriptors.get(tableName);
+    return tableDescriptor;
   }
   
   public HBaseColumn getColumn(String fieldName) {
@@ -69,18 +63,13 @@ public class HBaseMapping {
    * {@link HBaseMapping} using simple immutabilty.
    *
    */
-  public static class HBaseMappingBuilder {
-    private Map<String, HTableDescriptor> tableDescriptors 
-      = new HashMap<String, HTableDescriptor>();
-    private String tableName; 
+  public static class HBaseMappingBuilder { 
+    private Map<String, Map<String, HColumnDescriptor>> tableToFamilies = 
+      new HashMap<String, Map<String, HColumnDescriptor>>();
     private Map<String, HBaseColumn> columnMap = 
       new HashMap<String, HBaseColumn>();
     
-    public void addTable(String tableName) {
-      if(!tableDescriptors.containsKey(tableName)) {
-        tableDescriptors.put(tableName, new HTableDescriptor(tableName));
-      }
-    }
+    private String tableName;
     
     public String getTableName() {
       return tableName;
@@ -90,13 +79,28 @@ public class HBaseMapping {
       this.tableName = tableName;
     }
     
-    public void addColumnFamily(String tableName, String familyName,
+    public void addFamilyProps(String tableName, String familyName,
         String compression, String blockCache, String blockSize,
         String bloomFilter ,String maxVersions, String timeToLive, 
         String inMemory) {
       
-      HColumnDescriptor columnDescriptor = addColumnFamily(tableName, 
-          familyName);
+      // We keep track of all tables, because even though we
+      // only build a mapping for one table. We do this because of the way
+      // the mapping file is set up. 
+      // (First family properties are defined, whereafter columns are defined).
+      //
+      // HBaseMapping in fact does not need to support multiple tables,
+      // because a Store itself only supports a single table. (Every store 
+      // instance simply creates one mapping instance for itself).
+      //
+      // TODO A nice solution would be to redefine the mapping file structure.
+      // For example nest columns in families. Of course this would break compatibility.
+      
+      
+      Map<String, HColumnDescriptor> families = getOrCreateFamilies(tableName);;
+      
+      
+      HColumnDescriptor columnDescriptor = getOrCreateFamily(familyName, families);
       
       if(compression != null)
         columnDescriptor.setCompressionType(Algorithm.valueOf(compression));
@@ -112,42 +116,64 @@ public class HBaseMapping {
         columnDescriptor.setTimeToLive(Integer.parseInt(timeToLive));
       if(inMemory != null)
         columnDescriptor.setInMemory(Boolean.parseBoolean(inMemory));
-      
-      getTable(tableName).addFamily(columnDescriptor);
-    }
-    
-    public HTableDescriptor getTable(String tableName) {
-      return tableDescriptors.get(tableName);
     }
-    
-    public HColumnDescriptor addColumnFamily(String tableName, 
-        String familyName) {
-      HTableDescriptor tableDescriptor = getTable(tableName);
-      HColumnDescriptor columnDescriptor =  tableDescriptor.getFamily(
-          Bytes.toBytes(familyName));
-      if(columnDescriptor == null) {
-        columnDescriptor = new HColumnDescriptor(familyName);
-        tableDescriptor.addFamily(columnDescriptor);
-      }
-      return columnDescriptor;
+
+    public void addColumnFamily(String tableName, String familyName) {
+      Map<String, HColumnDescriptor> families = getOrCreateFamilies(tableName);
+      getOrCreateFamily(familyName, families);
     }
     
-    public void addField(String fieldName, String tableName, String family, 
-        String qualifier) {
+    public void addField(String fieldName, String family, String qualifier) {
       byte[] familyBytes = Bytes.toBytes(family);
       byte[] qualifierBytes = qualifier == null ? null : 
         Bytes.toBytes(qualifier);
       
-      HBaseColumn column = new HBaseColumn(tableName, familyBytes, 
-          qualifierBytes);
+      HBaseColumn column = new HBaseColumn(familyBytes, qualifierBytes);
       columnMap.put(fieldName, column);
     }
     
+
+    private HColumnDescriptor getOrCreateFamily(String familyName,
+        Map<String, HColumnDescriptor> families) {
+      HColumnDescriptor columnDescriptor = families.get(familyName);
+      if (columnDescriptor == null) {
+        columnDescriptor=new HColumnDescriptor(familyName);
+        families.put(familyName, columnDescriptor);
+      }
+      return columnDescriptor;
+    }
+
+    private Map<String, HColumnDescriptor> getOrCreateFamilies(String tableName) {
+      Map<String, HColumnDescriptor> families;
+      families = tableToFamilies.get(tableName);
+      if (families == null) {
+        families = new HashMap<String, HColumnDescriptor>();
+        tableToFamilies.put(tableName, families);
+      }
+      return families;
+    }
+    
+    public void renameTable(String oldName, String newName) {
+      Map<String, HColumnDescriptor> families = tableToFamilies.remove(oldName);
+      if (families == null) throw new IllegalArgumentException(oldName + " does not exist");
+      tableToFamilies.put(newName, families);
+    }
+    
     /**
      * @return A newly constructed mapping.
      */
     public HBaseMapping build() {
-      return new HBaseMapping(tableDescriptors, tableName, columnMap);
+      if (tableName == null) throw new IllegalStateException("tableName is not specified");
+      
+      Map<String, HColumnDescriptor> families = tableToFamilies.get(tableName);
+      if (families == null) throw new IllegalStateException("no families for table " + tableName);
+      
+      HTableDescriptor tableDescriptors = new HTableDescriptor(tableName);
+      for (HColumnDescriptor desc : families.values()) {
+        tableDescriptors.addFamily(desc);
+      }
+      return new HBaseMapping(tableDescriptors, columnMap);
     }
   }
+
 }
\ No newline at end of file

Modified: gora/trunk/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseStore.java
URL: http://svn.apache.org/viewvc/gora/trunk/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseStore.java?rev=1295608&r1=1295607&r2=1295608&view=diff
==============================================================================
--- gora/trunk/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseStore.java (original)
+++ gora/trunk/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseStore.java Thu Mar
 1 15:08:20 2012
@@ -130,17 +130,18 @@ implements Configurable {
       createSchema();
     }
 
-    table = new HBaseTableConnection(getConf(), mapping.getTableName(), true);
+    table = new HBaseTableConnection(getConf(), getSchemaName(), true);
   }
 
   @Override
   public String getSchemaName() {
+    //return the name of this table
     return mapping.getTableName();
   }
 
   @Override
   public void createSchema() throws IOException {
-    if(admin.tableExists(mapping.getTableName())) {
+    if(schemaExists()) {
       return;
     }
     HTableDescriptor tableDesc = mapping.getTable();
@@ -150,11 +151,11 @@ implements Configurable {
 
   @Override
   public void deleteSchema() throws IOException {
-    if(!admin.tableExists(mapping.getTableName())) {
+    if(!schemaExists()) {
       return;
     }
-    admin.disableTable(mapping.getTableName());
-    admin.deleteTable(mapping.getTableName());
+    admin.disableTable(getSchemaName());
+    admin.deleteTable(getSchemaName());
   }
 
   @Override
@@ -528,7 +529,6 @@ implements Configurable {
       List<Element> tableElements = root.getChildren("table");
       for(Element tableElement : tableElements) {
         String tableName = tableElement.getAttributeValue("name");
-        mappingBuilder.addTable(tableName);
 
         List<Element> fieldElements = tableElement.getChildren("family");
         for(Element fieldElement : fieldElements) {
@@ -540,8 +540,8 @@ implements Configurable {
           String maxVersions = fieldElement.getAttributeValue("maxVersions");
           String timeToLive  = fieldElement.getAttributeValue("timeToLive");
           String inMemory    = fieldElement.getAttributeValue("inMemory");
-
-          mappingBuilder.addColumnFamily(tableName, familyName, compression, 
+          
+          mappingBuilder.addFamilyProps(tableName, familyName, compression, 
               blockCache, blockSize, bloomFilter, maxVersions, timeToLive, 
               inMemory);
         }
@@ -554,22 +554,30 @@ implements Configurable {
             && classElement.getAttributeValue("name").equals(
                 persistentClass.getCanonicalName())) {
 
-          String tableName = getSchemaName(
-              classElement.getAttributeValue("table"), persistentClass);
-          mappingBuilder.addTable(tableName);
-          mappingBuilder.setTableName(tableName);
+          String tableNameFromMapping = classElement.getAttributeValue("table");
 
           List<Element> fields = classElement.getChildren("field");
           for(Element field:fields) {
             String fieldName =  field.getAttributeValue("name");
             String family =  field.getAttributeValue("family");
             String qualifier = field.getAttributeValue("qualifier");
-            mappingBuilder.addField(fieldName, mappingBuilder.getTableName(), 
-                family, qualifier);
-            mappingBuilder.addColumnFamily(mappingBuilder.getTableName(), 
-                family);//implicit family definition
+            mappingBuilder.addField(fieldName, family, qualifier);
+            mappingBuilder.addColumnFamily(tableNameFromMapping, family);
           }
+          
+          String tableName = getSchemaName(tableNameFromMapping, persistentClass);
+          
+          
+          if (!tableNameFromMapping.equals(tableName)) {
+            log.info("Keyclass and nameclass match but mismatching table names " 
+                + " mappingfile schema is '" + tableNameFromMapping 
+                + "' vs actual schema '" + tableName + "' , assuming they are the same.");
+            mappingBuilder.renameTable(tableNameFromMapping, tableName);
+          }
+          mappingBuilder.setTableName(tableName);
 
+          //we found a matching key and value class definition,
+          //do not continue on other class definitions
           break;
         }
       }



Mime
View raw message