incubator-hcatalog-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tof...@apache.org
Subject svn commit: r1304556 - in /incubator/hcatalog/branches/branch-0.4: ./ storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/ storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/ storage-handlers/hbase/src/test/org/apache/hcatalog/hba...
Date Fri, 23 Mar 2012 18:45:18 GMT
Author: toffer
Date: Fri Mar 23 18:45:18 2012
New Revision: 1304556

URL: http://svn.apache.org/viewvc?rev=1304556&view=rev
Log:
merged from trunk: HCATALOG-310 Turn current RM implementation into HBase Coprocessor (thw
via toffer)

Added:
    incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpoint.java
      - copied unchanged from r1304536, incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpoint.java
    incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpointClient.java
      - copied unchanged from r1304536, incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpointClient.java
    incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerProtocol.java
      - copied unchanged from r1304536, incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerProtocol.java
    incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/package-info.java
      - copied unchanged from r1304536, incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/package-info.java
    incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManagerEndpoint.java
      - copied unchanged from r1304536, incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManagerEndpoint.java
Modified:
    incubator/hcatalog/branches/branch-0.4/   (props changed)
    incubator/hcatalog/branches/branch-0.4/CHANGES.txt
    incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseRevisionManagerUtil.java
    incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManager.java
    incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerFactory.java
    incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/TableSnapshot.java
    incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/ManyMiniCluster.java
    incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/SkeletonHBaseTest.java

Propchange: incubator/hcatalog/branches/branch-0.4/
------------------------------------------------------------------------------
  Merged /incubator/hcatalog/trunk:r1304536

Modified: incubator/hcatalog/branches/branch-0.4/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/CHANGES.txt?rev=1304556&r1=1304555&r2=1304556&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/CHANGES.txt (original)
+++ incubator/hcatalog/branches/branch-0.4/CHANGES.txt Fri Mar 23 18:45:18 2012
@@ -56,6 +56,8 @@ Release 0.4.0 - Unreleased
   HCAT-240. Changes to HCatOutputFormat to make it use SerDes instead of StorageDriver (toffer)
 
   NEW FEATURES
+  HCAT-310 Turn current RM implementation into HBase Coprocessor (thw via toffer)
+
   HCAT-334 HCatalog should generate a POM file so it can be deployed to a maven repo (traviscrawford
via gates)
 
   HCAT-296 Hcatalog should be able talk to secure hbase server using hbase delegation tokens
(rohini via toffer)

Modified: incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseRevisionManagerUtil.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseRevisionManagerUtil.java?rev=1304556&r1=1304555&r2=1304556&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseRevisionManagerUtil.java
(original)
+++ incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseRevisionManagerUtil.java
Fri Mar 23 18:45:18 2012
@@ -24,13 +24,11 @@ import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Properties;
 import java.util.Set;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hive.hbase.HBaseSerDe;
 import org.apache.hcatalog.common.HCatConstants;
 import org.apache.hcatalog.common.HCatUtil;
@@ -40,7 +38,6 @@ import org.apache.hcatalog.hbase.snapsho
 import org.apache.hcatalog.hbase.snapshot.RevisionManagerFactory;
 import org.apache.hcatalog.hbase.snapshot.TableSnapshot;
 import org.apache.hcatalog.hbase.snapshot.Transaction;
-import org.apache.hcatalog.hbase.snapshot.ZKBasedRevisionManager;
 import org.apache.hcatalog.mapreduce.HCatTableInfo;
 import org.apache.hcatalog.mapreduce.InputJobInfo;
 import org.apache.hcatalog.mapreduce.OutputJobInfo;
@@ -126,37 +123,7 @@ class HBaseRevisionManagerUtil {
      * @throws IOException
      */
     static RevisionManager getOpenedRevisionManager(Configuration jobConf) throws IOException
{
-
-        Properties properties = new Properties();
-        String zkHostList = jobConf.get(HConstants.ZOOKEEPER_QUORUM);
-        int port = jobConf.getInt("hbase.zookeeper.property.clientPort",
-                HConstants.DEFAULT_ZOOKEPER_CLIENT_PORT);
-
-        if (zkHostList != null) {
-            String[] splits = zkHostList.split(",");
-            StringBuffer sb = new StringBuffer();
-            for (String split : splits) {
-                sb.append(split);
-                sb.append(':');
-                sb.append(port);
-                sb.append(',');
-            }
-
-            sb.deleteCharAt(sb.length() - 1);
-            properties.put(ZKBasedRevisionManager.HOSTLIST, sb.toString());
-        }
-        String dataDir = jobConf.get(ZKBasedRevisionManager.DATADIR);
-        if (dataDir != null) {
-            properties.put(ZKBasedRevisionManager.DATADIR, dataDir);
-        }
-        String rmClassName = jobConf.get(
-                RevisionManager.REVISION_MGR_IMPL_CLASS,
-                ZKBasedRevisionManager.class.getName());
-        properties.put(RevisionManager.REVISION_MGR_IMPL_CLASS, rmClassName);
-        RevisionManager revisionManger = RevisionManagerFactory
-                .getRevisionManager(properties);
-        revisionManger.open();
-        return revisionManger;
+      return RevisionManagerFactory.getOpenedRevisionManager(jobConf);
     }
 
     static void closeRevisionManagerQuietly(RevisionManager rm) {

Modified: incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManager.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManager.java?rev=1304556&r1=1304555&r2=1304556&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManager.java
(original)
+++ incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManager.java
Fri Mar 23 18:45:18 2012
@@ -25,8 +25,11 @@ import java.util.Properties;
  * This interface provides APIs for implementing revision management.
  */
 public interface RevisionManager {
-
-    public static final String REVISION_MGR_IMPL_CLASS = "revision.manager.impl.class";
+    /**
+     * Version property required by HBase to use this interface
+     * for CoprocessorProtocol / RPC.
+     */
+    public static final long VERSION = 1L; // do not change
 
     /**
      * Initialize the revision manager.

Modified: incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerFactory.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerFactory.java?rev=1304556&r1=1304555&r2=1304556&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerFactory.java
(original)
+++ incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerFactory.java
Fri Mar 23 18:45:18 2012
@@ -20,16 +20,26 @@ package org.apache.hcatalog.hbase.snapsh
 import java.io.IOException;
 import java.util.Properties;
 
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HConstants;
+
+/**
+ * Utility to instantiate the revision manager (not a true factory actually).
+ * Depends on HBase configuration to resolve ZooKeeper connection (when ZK is used).
+ */
 public class RevisionManagerFactory {
 
-   /**
-    * Gets an instance of revision manager.
-    *
-    * @param properties The properties required to created the revision manager.
-    * @return the revision manager An instance of revision manager.
-    * @throws IOException Signals that an I/O exception has occurred.
-    */
-   public static RevisionManager getRevisionManager(Properties properties) throws IOException{
+  public static final String REVISION_MGR_IMPL_CLASS = "revision.manager.impl.class";
+
+  /**
+   * Gets an instance of revision manager.
+   *
+   * @param properties The properties required to created the revision manager.
+   * @return the revision manager An instance of revision manager.
+   * @throws IOException Signals that an I/O exception has occurred.
+   */
+   private static RevisionManager getRevisionManager(String className, Properties properties)
throws IOException{
 
         RevisionManager revisionMgr;
         ClassLoader classLoader = Thread.currentThread()
@@ -37,14 +47,9 @@ public class RevisionManagerFactory {
         if (classLoader == null) {
             classLoader = RevisionManagerFactory.class.getClassLoader();
         }
-        String className = properties.getProperty(
-                RevisionManager.REVISION_MGR_IMPL_CLASS,
-                ZKBasedRevisionManager.class.getName());
         try {
-
-            @SuppressWarnings("unchecked")
-            Class<? extends RevisionManager> revisionMgrClass = (Class<? extends
RevisionManager>) Class
-                    .forName(className, true , classLoader);
+            Class<? extends RevisionManager> revisionMgrClass = Class
+                    .forName(className, true , classLoader).asSubclass(RevisionManager.class);
             revisionMgr = (RevisionManager) revisionMgrClass.newInstance();
             revisionMgr.initialize(properties);
         } catch (ClassNotFoundException e) {
@@ -67,4 +72,58 @@ public class RevisionManagerFactory {
         return revisionMgr;
     }
 
+   /**
+    * Internally used by endpoint implementation to instantiate from different configuration
setting.
+    * @param className
+    * @param conf
+    * @return
+    * @throws IOException
+    */
+   static RevisionManager getOpenedRevisionManager(String className, Configuration conf)
throws IOException {
+
+       Properties properties = new Properties();
+       String zkHostList = conf.get(HConstants.ZOOKEEPER_QUORUM);
+       int port = conf.getInt(HConstants.ZOOKEEPER_CLIENT_PORT,
+               HConstants.DEFAULT_ZOOKEPER_CLIENT_PORT);
+
+       if (zkHostList != null) {
+           String[] splits = zkHostList.split(",");
+           StringBuffer sb = new StringBuffer();
+           for (String split : splits) {
+               sb.append(split);
+               sb.append(':');
+               sb.append(port);
+               sb.append(',');
+           }
+
+           sb.deleteCharAt(sb.length() - 1);
+           properties.put(ZKBasedRevisionManager.HOSTLIST, sb.toString());
+       }
+       String dataDir = conf.get(ZKBasedRevisionManager.DATADIR);
+       if (dataDir != null) {
+           properties.put(ZKBasedRevisionManager.DATADIR, dataDir);
+       }
+       RevisionManager revisionMgr = RevisionManagerFactory
+               .getRevisionManager(className, properties);
+       if (revisionMgr instanceof Configurable) {
+         ((Configurable)revisionMgr).setConf(conf);
+       }
+       revisionMgr.open();
+       return revisionMgr;
+   }
+
+   /**
+    * Gets an instance of revision manager which is opened.
+    * The revision manager implementation can be specified as {@link #REVISION_MGR_IMPL_CLASS},
+    * default is {@link ZKBasedRevisionManager}.
+    * @param hbaseConf The HBase configuration.
+    * @return RevisionManager An instance of revision manager.
+    * @throws IOException
+    */
+   public static RevisionManager getOpenedRevisionManager(Configuration hbaseConf) throws
IOException {
+     String className = hbaseConf.get(RevisionManagerFactory.REVISION_MGR_IMPL_CLASS,
+         ZKBasedRevisionManager.class.getName());
+     return getOpenedRevisionManager(className, hbaseConf);
+   }
+
 }

Modified: incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/TableSnapshot.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/TableSnapshot.java?rev=1304556&r1=1304555&r2=1304556&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/TableSnapshot.java
(original)
+++ incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/TableSnapshot.java
Fri Mar 23 18:45:18 2012
@@ -17,6 +17,7 @@
  */
 package org.apache.hcatalog.hbase.snapshot;
 
+import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -24,7 +25,7 @@ import java.util.Map;
 /**
  * The snapshot for a table and a list of column families.
  */
-public class TableSnapshot {
+public class TableSnapshot implements Serializable {
 
     private String name;
 
@@ -35,6 +36,9 @@ public class TableSnapshot {
 
     public TableSnapshot(String name, Map<String, Long> cfRevMap, long latestRevision)
{
         this.name = name;
+        if (cfRevMap == null) {
+          throw new IllegalArgumentException("revision map cannot be null");
+        }
         this.cfRevisionMap = cfRevMap;
         this.latestRevision = latestRevision;
     }

Modified: incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/ManyMiniCluster.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/ManyMiniCluster.java?rev=1304556&r1=1304555&r2=1304556&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/ManyMiniCluster.java
(original)
+++ incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/ManyMiniCluster.java
Fri Mar 23 18:45:18 2012
@@ -254,8 +254,8 @@ public class ManyMiniCluster {
 
             hbaseConf.set("hbase.rootdir", hbaseRoot);
             hbaseConf.set("hbase.master", "local");
-            hbaseConf.setInt("hbase.zookeeper.property.clientPort", zookeeperPort);
-            hbaseConf.set("hbase.zookeeper.quorum", "127.0.0.1");
+            hbaseConf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, zookeeperPort);
+            hbaseConf.set(HConstants.ZOOKEEPER_QUORUM, "127.0.0.1");
             hbaseConf.setInt("hbase.master.port", findFreePort());
             hbaseConf.setInt("hbase.master.info.port", -1);
             hbaseConf.setInt("hbase.regionserver.port", findFreePort());
@@ -306,7 +306,7 @@ public class ManyMiniCluster {
         private File workDir;
         private int numTaskTrackers = 1;
         private JobConf jobConf;
-        private HBaseConfiguration hbaseConf;
+        private Configuration hbaseConf;
         private HiveConf hiveConf;
 
         private boolean miniMRClusterEnabled = true;
@@ -329,7 +329,7 @@ public class ManyMiniCluster {
             return this;
         }
 
-        public Builder hbaseConf(HBaseConfiguration hbaseConf) {
+        public Builder hbaseConf(Configuration hbaseConf) {
             this.hbaseConf = hbaseConf;
             return this;
         }

Modified: incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/SkeletonHBaseTest.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/SkeletonHBaseTest.java?rev=1304556&r1=1304555&r2=1304556&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/SkeletonHBaseTest.java
(original)
+++ incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/SkeletonHBaseTest.java
Fri Mar 23 18:45:18 2012
@@ -18,9 +18,18 @@
 
 package org.apache.hcatalog.hbase;
 
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
@@ -28,14 +37,6 @@ import org.apache.hadoop.hive.conf.HiveC
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 
-import java.io.File;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Random;
-import java.util.Set;
-
 /**
  * Base class for HBase Tests which need a mini cluster instance
  */
@@ -48,6 +49,12 @@ public abstract class SkeletonHBaseTest 
     protected static Map<String,Context> contextMap = new HashMap<String,Context>();
     protected static Set<String> tableNames = new HashSet<String>();
 
+    /**
+     * Allow tests to alter the default MiniCluster configuration.
+     * (requires static initializer block as all setup here is static)
+     */
+    protected static Configuration testConf = null;
+
     protected void createTable(String tableName, String[] families) {
         try {
             HBaseAdmin admin = new HBaseAdmin(getHbaseConf());
@@ -76,6 +83,7 @@ public abstract class SkeletonHBaseTest 
         return name;
     }
 
+
     /**
      * startup an hbase cluster instance before a test suite runs
      */
@@ -173,9 +181,13 @@ public abstract class SkeletonHBaseTest 
 
         public void start() {
             if(usageCount++ == 0) {
-                cluster = ManyMiniCluster.create(new File(testDir)).build();
+            	ManyMiniCluster.Builder b = ManyMiniCluster.create(new File(testDir));
+                if (testConf != null) {
+                   b.hbaseConf(HBaseConfiguration.create(testConf));
+                }
+                cluster = b.build();
                 cluster.start();
-                hbaseConf = cluster.getHBaseConf();
+                this.hbaseConf = cluster.getHBaseConf();
                 jobConf = cluster.getJobConf();
                 fileSystem = cluster.getFileSystem();
                 hiveConf = cluster.getHiveConf();



Mime
View raw message