incubator-hcatalog-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tof...@apache.org
Subject svn commit: r1340149 - in /incubator/hcatalog/trunk: ./ storage-handlers/hbase/ storage-handlers/hbase/conf/ storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/ storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/ storage-handlers...
Date Fri, 18 May 2012 16:10:04 GMT
Author: toffer
Date: Fri May 18 16:10:03 2012
New Revision: 1340149

URL: http://svn.apache.org/viewvc?rev=1340149&view=rev
Log:
HCAT-404 add separate configuration for revision manager (toffer)

Added:
    incubator/hcatalog/trunk/storage-handlers/hbase/conf/
    incubator/hcatalog/trunk/storage-handlers/hbase/conf/revision-manager-site.xml
    incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RMConstants.java
    incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerConfiguration.java
    incubator/hcatalog/trunk/storage-handlers/hbase/src/resources/
    incubator/hcatalog/trunk/storage-handlers/hbase/src/resources/revision-manager-default.xml
    incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManagerConfiguration.java
Modified:
    incubator/hcatalog/trunk/CHANGES.txt
    incubator/hcatalog/trunk/storage-handlers/hbase/build.xml
    incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseHCatStorageHandler.java
    incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManager.java
    incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpoint.java
    incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpointClient.java
    incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerFactory.java
    incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/ZKBasedRevisionManager.java
    incubator/hcatalog/trunk/storage-handlers/hbase/src/test/log4j.xml
    incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseBulkOutputFormat.java
    incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseDirectOutputFormat.java
    incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseHCatStorageHandler.java
    incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseInputFormat.java
    incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestSnapshots.java
    incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManager.java
    incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManagerEndpoint.java
    incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestZNodeSetUp.java

Modified: incubator/hcatalog/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/CHANGES.txt?rev=1340149&r1=1340148&r2=1340149&view=diff
==============================================================================
--- incubator/hcatalog/trunk/CHANGES.txt (original)
+++ incubator/hcatalog/trunk/CHANGES.txt Fri May 18 16:10:03 2012
@@ -98,6 +98,8 @@ Release 0.4.0 - Release May 16 2012
   HCAT-2 Support nested schema conversion between Hive an Pig (julienledem via hashutosh)
 
   IMPROVEMENTS
+  HCAT-404 add separate configuration for revision manager (toffer)
+
   HCAT-373 ProgressReporter should work with both old and new MR API (traviscrawford via toffer) 
 
   HCAT-68 Logging from HCat (avandana via toffer)

Modified: incubator/hcatalog/trunk/storage-handlers/hbase/build.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/build.xml?rev=1340149&r1=1340148&r2=1340149&view=diff
==============================================================================
--- incubator/hcatalog/trunk/storage-handlers/hbase/build.xml (original)
+++ incubator/hcatalog/trunk/storage-handlers/hbase/build.xml Fri May 18 16:10:03 2012
@@ -43,6 +43,7 @@
     <!-- build properties -->
     <property name="lib.dir" value="${basedir}/lib/" />
     <property name="src.dir"  location="${basedir}/src/java"/>
+    <property name="resources.dir"  location="${basedir}/src/resources"/>
     <property name="docs.src" value="${basedir}/src/docs"/>
     <property name="build.dir" value="${basedir}/build"/>
     <property name="build.classes" value="${build.dir}/classes" />
@@ -195,7 +196,7 @@
     -->
     <!-- Compile src files -->
     <target name="compile-src" depends="init">
-        <javac encoding="${build.encoding}" srcdir="${src.dir}:${basedir}/src/gen-java" excludes="${excludes}"
+        <javac encoding="${build.encoding}" srcdir="${src.dir}:${basedir}/src/gen-java:${basedir}/src/resources" excludes="${excludes}"
                includes="**/*.java" destdir="${build.classes}" debug="${javac.debug}"
                optimize="${javac.optimize}" target="${javac.version}"
                source="${javac.version}" deprecation="${javac.deprecation}"
@@ -207,6 +208,11 @@
 
     <!-- Build the jar -->
     <target name="jar" depends="compile-src">
+        <copy todir="${build.classes}" includeEmptyDirs="false">
+            <fileset dir="${resources.dir}">
+                <include name="*"/>
+            </fileset>
+        </copy>
         <jar jarfile="${build.dir}/${ant.project.name}/${handler.jar}" basedir="${build.classes}"/>
     </target>
 
@@ -306,6 +312,7 @@
     -->
     <target name="package" depends="jar, docs" description="Create an HCatalog release">
         <mkdir dir="${dist.dir}" />
+        <mkdir dir="${dist.dir}/conf" />
         <mkdir dir="${dist.dir}/lib" />
         <mkdir dir="${dist.dir}/docs" />
 
@@ -320,6 +327,9 @@
         <copy todir="${dist.dir}/docs">
             <fileset dir="${build.docs}" />
         </copy>
+        <copy todir="${dist.dir}/conf">
+            <fileset dir="${basedir}/conf" />
+        </copy>
 
         <antcall target="package-to-handlers"/>
     </target>
@@ -329,6 +339,7 @@
         <copy todir="${dist.handlers.dir}/hbase" includeEmptyDirs="true">
             <fileset dir="${dist.dir}">
               <include name="**/lib/hbase-storage-handler-*.jar"/>
+              <include name="**/conf/**/*"/>
             </fileset>
         </copy>
     </target>

Added: incubator/hcatalog/trunk/storage-handlers/hbase/conf/revision-manager-site.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/conf/revision-manager-site.xml?rev=1340149&view=auto
==============================================================================
--- incubator/hcatalog/trunk/storage-handlers/hbase/conf/revision-manager-site.xml (added)
+++ incubator/hcatalog/trunk/storage-handlers/hbase/conf/revision-manager-site.xml Fri May 18 16:10:03 2012
@@ -0,0 +1,25 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+</configuration>

Modified: incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseHCatStorageHandler.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseHCatStorageHandler.java?rev=1340149&r1=1340148&r2=1340149&view=diff
==============================================================================
--- incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseHCatStorageHandler.java (original)
+++ incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseHCatStorageHandler.java Fri May 18 16:10:03 2012
@@ -59,6 +59,7 @@ import org.apache.hcatalog.data.schema.H
 import org.apache.hcatalog.hbase.HBaseBulkOutputFormat.HBaseBulkOutputCommitter;
 import org.apache.hcatalog.hbase.HBaseDirectOutputFormat.HBaseDirectOutputCommitter;
 import org.apache.hcatalog.hbase.snapshot.RevisionManager;
+import org.apache.hcatalog.hbase.snapshot.RevisionManagerConfiguration;
 import org.apache.hcatalog.hbase.snapshot.Transaction;
 import org.apache.hcatalog.mapreduce.HCatOutputFormat;
 import org.apache.hcatalog.mapreduce.HCatTableInfo;
@@ -98,7 +99,7 @@ public class HBaseHCatStorageHandler ext
             jobProperties.put(TableInputFormat.INPUT_TABLE, qualifiedTableName);
 
             Configuration jobConf = getJobConf();
-            addHbaseResources(jobConf, jobProperties);
+            addResources(jobConf, jobProperties);
             JobConf copyOfConf = new JobConf(jobConf);
             HBaseConfiguration.addHbaseResources(copyOfConf);
             //Getting hbase delegation token in getInputSplits does not work with PIG. So need to
@@ -114,8 +115,10 @@ public class HBaseHCatStorageHandler ext
             String serSnapshot = (String) inputJobInfo.getProperties().get(
                     HBaseConstants.PROPERTY_TABLE_SNAPSHOT_KEY);
             if (serSnapshot == null) {
-                HCatTableSnapshot snapshot = HBaseRevisionManagerUtil.createSnapshot(copyOfConf,
-                        qualifiedTableName, tableInfo);
+                HCatTableSnapshot snapshot =
+                        HBaseRevisionManagerUtil.createSnapshot(
+                            RevisionManagerConfiguration.create(copyOfConf),
+                            qualifiedTableName, tableInfo);
                 jobProperties.put(HBaseConstants.PROPERTY_TABLE_SNAPSHOT_KEY,
                         HCatUtil.serialize(snapshot));
             }
@@ -146,7 +149,7 @@ public class HBaseHCatStorageHandler ext
             jobProperties.put(TableOutputFormat.OUTPUT_TABLE, qualifiedTableName);
 
             Configuration jobConf = getJobConf();
-            addHbaseResources(jobConf, jobProperties);
+            addResources(jobConf, jobProperties);
 
             Configuration copyOfConf = new Configuration(jobConf);
             HBaseConfiguration.addHbaseResources(copyOfConf);
@@ -155,7 +158,8 @@ public class HBaseHCatStorageHandler ext
                     HBaseConstants.PROPERTY_WRITE_TXN_KEY);
             Transaction txn = null;
             if (txnString == null) {
-                txn = HBaseRevisionManagerUtil.beginWriteTransaction(qualifiedTableName, tableInfo, copyOfConf);
+                txn = HBaseRevisionManagerUtil.beginWriteTransaction(qualifiedTableName, tableInfo,
+                        RevisionManagerConfiguration.create(copyOfConf));
                 String serializedTxn = HCatUtil.serialize(txn);
                 outputJobInfo.getProperties().setProperty(HBaseConstants.PROPERTY_WRITE_TXN_KEY,
                         serializedTxn);
@@ -477,7 +481,7 @@ public class HBaseHCatStorageHandler ext
         //configureOutputJobProperties, we take care of adding the default properties
         //that are not already present. TODO: Change to a copy for jobs after HCAT-308 is fixed.
         jobConf = conf;
-        hbaseConf = HBaseConfiguration.create(conf);
+        hbaseConf = RevisionManagerConfiguration.create(HBaseConfiguration.create(conf));
     }
 
     private void checkDeleteTable(Table table) throws MetaException {
@@ -540,10 +544,11 @@ public class HBaseHCatStorageHandler ext
      * @param jobConf Job configuration
      * @param newJobProperties  Map to which new properties should be added
      */
-    private void addHbaseResources(Configuration jobConf,
-            Map<String, String> newJobProperties) {
+    private void addResources(Configuration jobConf,
+                              Map<String, String> newJobProperties) {
         Configuration conf = new Configuration(false);
         HBaseConfiguration.addHbaseResources(conf);
+        RevisionManagerConfiguration.addResources(conf);
         for (Entry<String, String> entry : conf) {
             if (jobConf.get(entry.getKey()) == null)
                 newJobProperties.put(entry.getKey(), entry.getValue());

Added: incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RMConstants.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RMConstants.java?rev=1340149&view=auto
==============================================================================
--- incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RMConstants.java (added)
+++ incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RMConstants.java Fri May 18 16:10:03 2012
@@ -0,0 +1,11 @@
+package org.apache.hcatalog.hbase.snapshot;
+
+public class RMConstants {
+  public static final String REVISION_MGR_ENDPOINT_IMPL_CLASS = "revision.manager.endpoint.impl.class";
+
+  public static final String WRITE_TRANSACTION_TIMEOUT = "revision.manager.writeTxn.timeout";
+
+  public static final String ZOOKEEPER_HOSTLIST = "revision.manager.zk.hostList";
+
+  public static final String ZOOKEEPER_DATADIR = "revision.manager.zk.dataDir";
+}

Modified: incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManager.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManager.java?rev=1340149&r1=1340148&r2=1340149&view=diff
==============================================================================
--- incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManager.java (original)
+++ incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManager.java Fri May 18 16:10:03 2012
@@ -17,6 +17,8 @@
  */
 package org.apache.hcatalog.hbase.snapshot;
 
+import org.apache.hadoop.conf.Configuration;
+
 import java.io.IOException;
 import java.util.List;
 import java.util.Properties;
@@ -34,7 +36,7 @@ public interface RevisionManager {
     /**
      * Initialize the revision manager.
      */
-    public void initialize(Properties properties);
+    public void initialize(Configuration conf);
 
     /**
      * Opens the revision manager.

Added: incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerConfiguration.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerConfiguration.java?rev=1340149&view=auto
==============================================================================
--- incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerConfiguration.java (added)
+++ incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerConfiguration.java Fri May 18 16:10:03 2012
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hcatalog.hbase.snapshot;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+
+public class RevisionManagerConfiguration {
+
+
+
+  public static Configuration addResources(Configuration conf) {
+    conf.addDefaultResource("revision-manager-default.xml");
+    conf.addResource("revision-manager-site.xml");
+    return conf;
+  }
+
+  /**
+   * Creates a Configuration with Revision Manager resources
+   * @return a Configuration with Revision Manager resources
+   */
+  public static Configuration create() {
+    Configuration conf = new Configuration();
+    return addResources(conf);
+  }
+
+  /**
+   * Creates a clone of passed configuration.
+   * @param that Configuration to clone.
+   * @return a Configuration created with the revision-manager-*.xml files plus
+   * the given configuration.
+   */
+  public static Configuration create(final Configuration that) {
+    Configuration conf = create();
+    //we need to merge things instead of doing new Configuration(that)
+    //because of a bug in Configuration wherein the config
+    //set on the MR fronted will get loaded on the backend as resouce called job.xml
+    //hence adding resources on the backed could potentially overwrite properties
+    //set on the frontend which we shouldn't be doing here
+    HBaseConfiguration.merge(conf, that);
+    return conf;
+  }
+}

Modified: incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpoint.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpoint.java?rev=1340149&r1=1340148&r2=1340149&view=diff
==============================================================================
--- incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpoint.java (original)
+++ incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpoint.java Fri May 18 16:10:03 2012
@@ -38,7 +38,6 @@ public class RevisionManagerEndpoint ext
 
   private static final Logger LOGGER =
 		      LoggerFactory.getLogger(RevisionManagerEndpoint.class.getName());
-  public static final String REVISION_MGR_ENDPOINT_IMPL_CLASS = "revision.manager.endpoint.impl.class";
   
   private RevisionManager rmImpl = null;
 
@@ -46,9 +45,10 @@ public class RevisionManagerEndpoint ext
   public void start(CoprocessorEnvironment env) {
     super.start(env);
     try {
-      Configuration conf = env.getConfiguration();
-      String className = conf.get(REVISION_MGR_ENDPOINT_IMPL_CLASS,
+      Configuration conf = RevisionManagerConfiguration.create(env.getConfiguration());
+      String className = conf.get(RMConstants.REVISION_MGR_ENDPOINT_IMPL_CLASS,
           ZKBasedRevisionManager.class.getName());
+      LOGGER.debug("Using Revision Manager implementation: {}",className);
       rmImpl = RevisionManagerFactory.getOpenedRevisionManager(className, conf);
     } catch (IOException e) {
       LOGGER.error("Failed to initialize revision manager", e);
@@ -68,7 +68,7 @@ public class RevisionManagerEndpoint ext
   }
 
   @Override
-  public void initialize(Properties properties) {
+  public void initialize(Configuration conf) {
     // do nothing, HBase controls life cycle
   }
 

Modified: incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpointClient.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpointClient.java?rev=1340149&r1=1340148&r2=1340149&view=diff
==============================================================================
--- incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpointClient.java (original)
+++ incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpointClient.java Fri May 18 16:10:03 2012
@@ -30,7 +30,7 @@ public class RevisionManagerEndpointClie
   }
 
   @Override
-  public void initialize(Properties properties) {
+  public void initialize(Configuration conf) {
     // do nothing
   }
 
@@ -41,7 +41,7 @@ public class RevisionManagerEndpointClie
     // conf.set("hbase.ipc.client.connect.max.retries", "0");
     // conf.setInt(HConstants.HBASE_CLIENT_RPC_MAXATTEMPTS, 1);
     clonedConf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1); // do not retry RPC
-    HTable table = new HTable(clonedConf, HConstants.META_TABLE_NAME);
+    HTable table = new HTable(clonedConf, HConstants.ROOT_TABLE_NAME);
     rmProxy = table.coprocessorProxy(RevisionManagerProtocol.class,
         Bytes.toBytes("anyRow"));
     rmProxy.open();

Modified: incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerFactory.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerFactory.java?rev=1340149&r1=1340148&r2=1340149&view=diff
==============================================================================
--- incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerFactory.java (original)
+++ incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerFactory.java Fri May 18 16:10:03 2012
@@ -35,11 +35,11 @@ public class RevisionManagerFactory {
   /**
    * Gets an instance of revision manager.
    *
-   * @param properties The properties required to created the revision manager.
+   * @param conf The configuration required to created the revision manager.
    * @return the revision manager An instance of revision manager.
    * @throws IOException Signals that an I/O exception has occurred.
    */
-   private static RevisionManager getRevisionManager(String className, Properties properties) throws IOException{
+   private static RevisionManager getRevisionManager(String className, Configuration conf) throws IOException{
 
         RevisionManager revisionMgr;
         ClassLoader classLoader = Thread.currentThread()
@@ -51,7 +51,7 @@ public class RevisionManagerFactory {
             Class<? extends RevisionManager> revisionMgrClass = Class
                     .forName(className, true , classLoader).asSubclass(RevisionManager.class);
             revisionMgr = (RevisionManager) revisionMgrClass.newInstance();
-            revisionMgr.initialize(properties);
+            revisionMgr.initialize(conf);
         } catch (ClassNotFoundException e) {
             throw new IOException(
                     "The implementation class of revision manager not found.",
@@ -81,30 +81,7 @@ public class RevisionManagerFactory {
     */
    static RevisionManager getOpenedRevisionManager(String className, Configuration conf) throws IOException {
 
-       Properties properties = new Properties();
-       String zkHostList = conf.get(HConstants.ZOOKEEPER_QUORUM);
-       int port = conf.getInt(HConstants.ZOOKEEPER_CLIENT_PORT,
-               HConstants.DEFAULT_ZOOKEPER_CLIENT_PORT);
-
-       if (zkHostList != null) {
-           String[] splits = zkHostList.split(",");
-           StringBuffer sb = new StringBuffer();
-           for (String split : splits) {
-               sb.append(split);
-               sb.append(':');
-               sb.append(port);
-               sb.append(',');
-           }
-
-           sb.deleteCharAt(sb.length() - 1);
-           properties.put(ZKBasedRevisionManager.HOSTLIST, sb.toString());
-       }
-       String dataDir = conf.get(ZKBasedRevisionManager.DATADIR);
-       if (dataDir != null) {
-           properties.put(ZKBasedRevisionManager.DATADIR, dataDir);
-       }
-       RevisionManager revisionMgr = RevisionManagerFactory
-               .getRevisionManager(className, properties);
+       RevisionManager revisionMgr = RevisionManagerFactory.getRevisionManager(className, conf);
        if (revisionMgr instanceof Configurable) {
          ((Configurable)revisionMgr).setConf(conf);
        }
@@ -116,14 +93,14 @@ public class RevisionManagerFactory {
     * Gets an instance of revision manager which is opened.
     * The revision manager implementation can be specified as {@link #REVISION_MGR_IMPL_CLASS},
     * default is {@link ZKBasedRevisionManager}.
-    * @param hbaseConf The HBase configuration.
+    * @param conf revision manager configuration
     * @return RevisionManager An instance of revision manager.
     * @throws IOException
     */
-   public static RevisionManager getOpenedRevisionManager(Configuration hbaseConf) throws IOException {
-     String className = hbaseConf.get(RevisionManagerFactory.REVISION_MGR_IMPL_CLASS,
+   public static RevisionManager getOpenedRevisionManager(Configuration conf) throws IOException {
+     String className = conf.get(RevisionManagerFactory.REVISION_MGR_IMPL_CLASS,
          ZKBasedRevisionManager.class.getName());
-     return getOpenedRevisionManager(className, hbaseConf);
+     return getOpenedRevisionManager(className, conf);
    }
 
 }

Modified: incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/ZKBasedRevisionManager.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/ZKBasedRevisionManager.java?rev=1340149&r1=1340148&r2=1340149&view=diff
==============================================================================
--- incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/ZKBasedRevisionManager.java (original)
+++ incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/ZKBasedRevisionManager.java Fri May 18 16:10:03 2012
@@ -25,6 +25,8 @@ import java.util.Properties;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HConstants;
 import org.apache.hcatalog.hbase.snapshot.lock.LockListener;
 import org.apache.hcatalog.hbase.snapshot.lock.WriteLock;
 import org.apache.zookeeper.CreateMode;
@@ -36,27 +38,37 @@ import org.apache.zookeeper.ZooDefs.Ids;
  */
 public class ZKBasedRevisionManager implements RevisionManager{
 
-    public static final String HOSTLIST = "revision.manager.zk.HostList";
-    public static final String DATADIR = "revision.manager.zk.DataDir";
-    public static final String DEFAULT_DATADIR = "/revision-management";
-    public static final String DEFAULT_HOSTLIST = "localhost:2181";
-    private static  int DEFAULT_WRITE_TRANSACTION_TIMEOUT = 14400000;
     private static final Log LOG = LogFactory.getLog(ZKBasedRevisionManager.class);
     private String zkHostList;
     private String baseDir;
     private ZKUtil zkUtil;
+    private long writeTxnTimeout;
 
 
     /*
      * @see org.apache.hcatalog.hbase.snapshot.RevisionManager#initialize()
      */
     @Override
-    public void initialize(Properties properties) {
-        this.zkHostList = properties.getProperty(
-                ZKBasedRevisionManager.HOSTLIST,
-                ZKBasedRevisionManager.DEFAULT_HOSTLIST);
-        this.baseDir = properties.getProperty(ZKBasedRevisionManager.DATADIR,
-                ZKBasedRevisionManager.DEFAULT_DATADIR);
+    public void initialize(Configuration conf) {
+        conf = new Configuration(conf);
+        if (conf.get(RMConstants.ZOOKEEPER_HOSTLIST) == null) {
+           String zkHostList = conf.get(HConstants.ZOOKEEPER_QUORUM);
+           int port = conf.getInt(HConstants.ZOOKEEPER_CLIENT_PORT,
+                   HConstants.DEFAULT_ZOOKEPER_CLIENT_PORT);
+           String[] splits = zkHostList.split(",");
+           StringBuffer sb = new StringBuffer();
+           for (String split : splits) {
+               sb.append(split);
+               sb.append(':');
+               sb.append(port);
+               sb.append(',');
+           }
+           sb.deleteCharAt(sb.length() - 1);
+           conf.set(RMConstants.ZOOKEEPER_HOSTLIST, sb.toString());
+        }
+        this.zkHostList = conf.get(RMConstants.ZOOKEEPER_HOSTLIST);
+        this.baseDir = conf.get(RMConstants.ZOOKEEPER_DATADIR);
+        this.writeTxnTimeout = Long.parseLong(conf.get(RMConstants.WRITE_TRANSACTION_TIMEOUT));
     }
 
     /**
@@ -118,7 +130,7 @@ public class ZKBasedRevisionManager impl
         if (keepAlive != -1) {
             transaction.setKeepAlive(keepAlive);
         } else {
-            transaction.setKeepAlive(DEFAULT_WRITE_TRANSACTION_TIMEOUT);
+            transaction.setKeepAlive(writeTxnTimeout);
         }
 
         refreshTransactionList(transaction.getTableName());

Added: incubator/hcatalog/trunk/storage-handlers/hbase/src/resources/revision-manager-default.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/src/resources/revision-manager-default.xml?rev=1340149&view=auto
==============================================================================
--- incubator/hcatalog/trunk/storage-handlers/hbase/src/resources/revision-manager-default.xml (added)
+++ incubator/hcatalog/trunk/storage-handlers/hbase/src/resources/revision-manager-default.xml Fri May 18 16:10:03 2012
@@ -0,0 +1,45 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+  <property>
+    <name>revision.manager.impl.class</name>
+    <value>org.apache.hcatalog.hbase.snapshot.ZKBasedRevisionManager</value>
+    <description>
+        Which revision manager implementation to use.
+    </description>
+  </property>
+  <property>
+    <name>revision.manager.endpoint.impl.class</name>
+    <value>org.apache.hcatalog.hbase.snapshot.ZKBasedRevisionManager</value>
+    <description> </description>
+  </property>
+  <property>
+    <name>revision.manager.zk.dataDir</name>
+    <value>/revision-management</value>
+    <description>zookeeper base directory</description>
+  </property>
+  <property>
+    <name>revision.manager.writeTxn.timeout</name>
+    <value>14400000</value>
+    <description>timeout for write transactions in seconds</description>
+  </property>
+</configuration>

Modified: incubator/hcatalog/trunk/storage-handlers/hbase/src/test/log4j.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/src/test/log4j.xml?rev=1340149&r1=1340148&r2=1340149&view=diff
==============================================================================
--- incubator/hcatalog/trunk/storage-handlers/hbase/src/test/log4j.xml (original)
+++ incubator/hcatalog/trunk/storage-handlers/hbase/src/test/log4j.xml Fri May 18 16:10:03 2012
@@ -27,7 +27,7 @@
 
 
     <root>
-        <priority value ="INFO" />
+        <priority value ="DEBUG" />
         <appender-ref ref="ConsoleAppender" />
     </root>
     <!--<logger name="org.apache.hadoop.mapred">-->

Modified: incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseBulkOutputFormat.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseBulkOutputFormat.java?rev=1340149&r1=1340148&r2=1340149&view=diff
==============================================================================
--- incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseBulkOutputFormat.java (original)
+++ incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseBulkOutputFormat.java Fri May 18 16:10:03 2012
@@ -24,6 +24,7 @@ import org.apache.hadoop.conf.Configurat
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
@@ -59,6 +60,7 @@ import org.apache.hcatalog.hbase.TestHBa
 import org.apache.hcatalog.hbase.TestHBaseDirectOutputFormat.MapWriteAbortTransaction;
 import org.apache.hcatalog.hbase.snapshot.FamilyRevision;
 import org.apache.hcatalog.hbase.snapshot.RevisionManager;
+import org.apache.hcatalog.hbase.snapshot.RevisionManagerConfiguration;
 import org.apache.hcatalog.hbase.snapshot.TableSnapshot;
 import org.apache.hcatalog.hbase.snapshot.Transaction;
 import org.apache.hcatalog.mapreduce.HCatInputFormat;
@@ -100,6 +102,9 @@ public class TestHBaseBulkOutputFormat e
         for (Map.Entry<String, String> el : getJobConf())
             allConf.set(el.getKey(), el.getValue());
 
+        HBaseConfiguration.merge(
+                allConf,
+                RevisionManagerConfiguration.create());
         SessionState.start(new CliSessionState(allConf));
         hcatDriver = new HCatDriver();
     }

Modified: incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseDirectOutputFormat.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseDirectOutputFormat.java?rev=1340149&r1=1340148&r2=1340149&view=diff
==============================================================================
--- incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseDirectOutputFormat.java (original)
+++ incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseDirectOutputFormat.java Fri May 18 16:10:03 2012
@@ -22,6 +22,7 @@ import org.apache.hadoop.conf.Configurat
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
@@ -55,6 +56,7 @@ import org.apache.hcatalog.data.HCatReco
 import org.apache.hcatalog.data.schema.HCatSchema;
 import org.apache.hcatalog.hbase.snapshot.FamilyRevision;
 import org.apache.hcatalog.hbase.snapshot.RevisionManager;
+import org.apache.hcatalog.hbase.snapshot.RevisionManagerConfiguration;
 import org.apache.hcatalog.hbase.snapshot.TableSnapshot;
 import org.apache.hcatalog.hbase.snapshot.Transaction;
 import org.apache.hcatalog.mapreduce.HCatInputFormat;
@@ -67,9 +69,11 @@ import java.io.IOException;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotSame;
 import static org.junit.Assert.assertTrue;
 
 /**
@@ -92,7 +96,9 @@ public class TestHBaseDirectOutputFormat
             allConf.set(el.getKey(), el.getValue());
         for (Map.Entry<String, String> el : getJobConf())
             allConf.set(el.getKey(), el.getValue());
-
+        HBaseConfiguration.merge(
+                allConf,
+                RevisionManagerConfiguration.create());
         SessionState.start(new CliSessionState(allConf));
         hcatDriver = new HCatDriver();
     }
@@ -330,8 +336,11 @@ public class TestHBaseDirectOutputFormat
         Scan scan = new Scan();
         scan.addFamily(familyNameBytes);
         ResultScanner scanner = table.getScanner(scan);
-        int index = 0;
+        int count = 0;
         for (Result result : scanner) {
+            String key = Bytes.toString(result.getRow());
+            assertNotSame(MapWriteAbortTransaction.failedKey, key);
+            int index = Integer.parseInt(key)-1;
             String vals[] = data[index].toString().split(",");
             for (int i = 1; i < vals.length; i++) {
                 String pair[] = vals[i].split(":");
@@ -341,9 +350,9 @@ public class TestHBaseDirectOutputFormat
                 assertEquals(1l, result.getColumn(familyNameBytes, Bytes.toBytes(pair[0])).get(0)
                         .getTimestamp());
             }
-            index++;
+            count++;
         }
-        assertEquals(data.length - 1, index);
+        assertEquals(data.length - 1, count);
 
         // verify that the inputformat returns empty results.
         Path outputDir = new Path(getTestDir(),
@@ -441,6 +450,8 @@ public class TestHBaseDirectOutputFormat
     }
 
     static class MapWriteAbortTransaction extends Mapper<LongWritable, Text, BytesWritable, HCatRecord> {
+        public static String failedKey;
+        private static int count = 0;
 
         @Override
         public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
@@ -449,14 +460,19 @@ public class TestHBaseDirectOutputFormat
             HCatSchema schema = jobInfo.getOutputSchema();
             String vals[] = value.toString().split(",");
             record.setInteger("key", schema, Integer.parseInt(vals[0]));
-            if (vals[0].equals("3")) {
-                throw new IOException("Failing map to test abort");
-            }
-            for (int i = 1; i < vals.length; i++) {
-                String pair[] = vals[i].split(":");
-                record.set(pair[0], schema, pair[1]);
+            synchronized (MapWriteAbortTransaction.class) {
+                if (count == 2) {
+                    failedKey = vals[0];
+                    throw new IOException("Failing map to test abort");
+                }
+                for (int i = 1; i < vals.length; i++) {
+                    String pair[] = vals[i].split(":");
+                    record.set(pair[0], schema, pair[1]);
+                }
+                context.write(null, record);
+                count++;
             }
-            context.write(null, record);
+
         }
 
     }

Modified: incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseHCatStorageHandler.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseHCatStorageHandler.java?rev=1340149&r1=1340148&r2=1340149&view=diff
==============================================================================
--- incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseHCatStorageHandler.java (original)
+++ incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseHCatStorageHandler.java Fri May 18 16:10:03 2012
@@ -24,6 +24,7 @@ import java.net.URI;
 import java.util.Map;
 
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
@@ -37,6 +38,7 @@ import org.apache.hadoop.hive.ql.session
 import org.apache.hcatalog.cli.HCatDriver;
 import org.apache.hcatalog.cli.SemanticAnalysis.HCatSemanticAnalyzer;
 import org.apache.hcatalog.hbase.snapshot.RevisionManager;
+import org.apache.hcatalog.hbase.snapshot.RevisionManagerConfiguration;
 import org.apache.zookeeper.KeeperException.NoNodeException;
 import org.junit.Test;
 
@@ -58,12 +60,14 @@ public class TestHBaseHCatStorageHandler
         hcatConf.set(ConfVars.METASTOREWAREHOUSE.varname, whPath.toString());
 
         //Add hbase properties
-
         for (Map.Entry<String, String> el : getHbaseConf()) {
             if (el.getKey().startsWith("hbase.")) {
                 hcatConf.set(el.getKey(), el.getValue());
             }
         }
+        HBaseConfiguration.merge(
+                hcatConf,
+                RevisionManagerConfiguration.create());
 
         SessionState.start(new CliSessionState(hcatConf));
         hcatDriver = new HCatDriver();
@@ -87,7 +91,7 @@ public class TestHBaseHCatStorageHandler
 
         assertTrue(doesTableExist);
 
-        RevisionManager rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(getHbaseConf());
+        RevisionManager rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(hcatConf);
         rm.open();
         //Should be able to successfully query revision manager
         rm.getAbortedWriteTransactions("test_table", "cf1");
@@ -123,7 +127,7 @@ public class TestHBaseHCatStorageHandler
 
         assertTrue(doesTableExist);
 
-        RevisionManager rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(getHbaseConf());
+        RevisionManager rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(hcatConf);
         rm.open();
         //Should be able to successfully query revision manager
         rm.getAbortedWriteTransactions("test_table", "cf1");
@@ -159,7 +163,7 @@ public class TestHBaseHCatStorageHandler
 
         assertTrue(doesTableExist);
 
-        RevisionManager rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(getHbaseConf());
+        RevisionManager rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(hcatConf);
         rm.open();
         //Should be able to successfully query revision manager
         rm.getAbortedWriteTransactions("CaseSensitiveTable", "cf1");

Modified: incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseInputFormat.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseInputFormat.java?rev=1340149&r1=1340148&r2=1340149&view=diff
==============================================================================
--- incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseInputFormat.java (original)
+++ incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseInputFormat.java Fri May 18 16:10:03 2012
@@ -32,6 +32,7 @@ import java.util.Map.Entry;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HTable;
@@ -66,6 +67,7 @@ import org.apache.hcatalog.data.HCatReco
 import org.apache.hcatalog.data.schema.HCatFieldSchema;
 import org.apache.hcatalog.data.schema.HCatSchema;
 import org.apache.hcatalog.hbase.snapshot.RevisionManager;
+import org.apache.hcatalog.hbase.snapshot.RevisionManagerConfiguration;
 import org.apache.hcatalog.hbase.snapshot.Transaction;
 import org.apache.hcatalog.mapreduce.HCatInputFormat;
 import org.apache.hcatalog.mapreduce.InputJobInfo;
@@ -80,13 +82,39 @@ public class TestHBaseInputFormat extend
     private final byte[] QUALIFIER1 = Bytes.toBytes("testQualifier1");
     private final byte[] QUALIFIER2 = Bytes.toBytes("testQualifier2");
 
+   public TestHBaseInputFormat() throws Exception {
+        hcatConf = getHiveConf();
+        hcatConf.set(ConfVars.SEMANTIC_ANALYZER_HOOK.varname,
+                HCatSemanticAnalyzer.class.getName());
+        URI fsuri = getFileSystem().getUri();
+        Path whPath = new Path(fsuri.getScheme(), fsuri.getAuthority(),
+                getTestDir());
+        hcatConf.set(HiveConf.ConfVars.HADOOPFS.varname, fsuri.toString());
+        hcatConf.set(ConfVars.METASTOREWAREHOUSE.varname, whPath.toString());
+
+        //Add hbase properties
+
+        for (Map.Entry<String, String> el : getHbaseConf()) {
+            if (el.getKey().startsWith("hbase.")) {
+                hcatConf.set(el.getKey(), el.getValue());
+            }
+        }
+        HBaseConfiguration.merge(hcatConf,
+               RevisionManagerConfiguration.create());
+
+
+        SessionState.start(new CliSessionState(hcatConf));
+        hcatDriver = new HCatDriver();
+
+    }
+
     private List<Put> generatePuts(int num, String tableName) throws IOException {
 
         List<String> columnFamilies = Arrays.asList("testFamily");
         RevisionManager rm = null;
         List<Put> myPuts;
         try {
-            rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(getHbaseConf());
+            rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(hcatConf);
             rm.open();
             myPuts = new ArrayList<Put>();
             for (int i = 1; i <= num; i++) {
@@ -106,29 +134,6 @@ public class TestHBaseInputFormat extend
         return myPuts;
     }
 
-   private void Initialize() throws Exception {
-        hcatConf = getHiveConf();
-        hcatConf.set(ConfVars.SEMANTIC_ANALYZER_HOOK.varname,
-                HCatSemanticAnalyzer.class.getName());
-        URI fsuri = getFileSystem().getUri();
-        Path whPath = new Path(fsuri.getScheme(), fsuri.getAuthority(),
-                getTestDir());
-        hcatConf.set(HiveConf.ConfVars.HADOOPFS.varname, fsuri.toString());
-        hcatConf.set(ConfVars.METASTOREWAREHOUSE.varname, whPath.toString());
-
-        //Add hbase properties
-
-        for (Map.Entry<String, String> el : getHbaseConf()) {
-            if (el.getKey().startsWith("hbase.")) {
-                hcatConf.set(el.getKey(), el.getValue());
-            }
-        }
-
-        SessionState.start(new CliSessionState(hcatConf));
-        hcatDriver = new HCatDriver();
-
-    }
-
    private void populateHBaseTable(String tName, int revisions) throws IOException {
         List<Put> myPuts = generatePuts(revisions, tName);
         HTable table = new HTable(getHbaseConf(), Bytes.toBytes(tName));
@@ -142,7 +147,7 @@ public class TestHBaseInputFormat extend
         List<Put> myPuts = new ArrayList<Put>();
         long revision;
         try {
-            rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(getHbaseConf());
+            rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(hcatConf);
             rm.open();
             Transaction tsx = rm.beginWriteTransaction(tName, columnFamilies);
 
@@ -171,7 +176,6 @@ public class TestHBaseInputFormat extend
 
     @Test
     public void TestHBaseTableReadMR() throws Exception {
-        Initialize();
         String tableName = newTableName("MyTable");
         String databaseName = newTableName("MyDatabase");
         //Table name will be lower case unless specified by hbase.table.name property
@@ -243,7 +247,6 @@ public class TestHBaseInputFormat extend
     @Test
     public void TestHBaseTableProjectionReadMR() throws Exception {
 
-        Initialize();
         String tableName = newTableName("MyTable");
         //Table name as specified by hbase.table.name property
         String hbaseTableName = "MyDB_" + tableName;
@@ -304,7 +307,6 @@ public class TestHBaseInputFormat extend
     @Test
     public void TestHBaseInputFormatProjectionReadMR() throws Exception {
 
-        Initialize();
         String tableName = newTableName("mytable");
         String tableQuery = "CREATE TABLE " + tableName
                               + "(key string, testqualifier1 string, testqualifier2 string) STORED BY " +
@@ -377,7 +379,6 @@ public class TestHBaseInputFormat extend
 
     @Test
     public void TestHBaseTableIgnoreAbortedTransactions() throws Exception {
-        Initialize();
         String tableName = newTableName("mytable");
         String tableQuery = "CREATE TABLE " + tableName
                               + "(key string, testqualifier1 string, testqualifier2 string) STORED BY " +
@@ -437,7 +438,6 @@ public class TestHBaseInputFormat extend
 
     @Test
     public void TestHBaseTableIgnoreAbortedAndRunningTransactions() throws Exception {
-        Initialize();
         String tableName = newTableName("mytable");
         String tableQuery = "CREATE TABLE " + tableName
                               + "(key string, testqualifier1 string, testqualifier2 string) STORED BY " +

Modified: incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestSnapshots.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestSnapshots.java?rev=1340149&r1=1340148&r2=1340149&view=diff
==============================================================================
--- incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestSnapshots.java (original)
+++ incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestSnapshots.java Fri May 18 16:10:03 2012
@@ -26,6 +26,7 @@ import java.util.Map;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hive.cli.CliSessionState;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
@@ -36,6 +37,7 @@ import org.apache.hcatalog.cli.HCatDrive
 import org.apache.hcatalog.cli.SemanticAnalysis.HCatSemanticAnalyzer;
 import org.apache.hcatalog.common.HCatConstants;
 import org.apache.hcatalog.common.HCatUtil;
+import org.apache.hcatalog.hbase.snapshot.RevisionManagerConfiguration;
 import org.apache.hcatalog.hbase.snapshot.TableSnapshot;
 import org.apache.hcatalog.mapreduce.InitializeInput;
 import org.apache.hcatalog.mapreduce.InputJobInfo;

Modified: incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManager.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManager.java?rev=1340149&r1=1340148&r2=1340149&view=diff
==============================================================================
--- incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManager.java (original)
+++ incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManager.java Fri May 18 16:10:03 2012
@@ -23,8 +23,8 @@ import static org.junit.Assert.assertTru
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.List;
-import java.util.Properties;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hcatalog.hbase.SkeletonHBaseTest;
 import org.apache.hcatalog.hbase.snapshot.transaction.thrift.*;
 import org.apache.zookeeper.KeeperException;
@@ -84,11 +84,10 @@ public class TestRevisionManager extends
             sb.append(port);
         }
 
-        Properties props = new Properties();
-        props.put(ZKBasedRevisionManager.HOSTLIST, sb.toString());
-        props.put(ZKBasedRevisionManager.DATADIR, "/rm_base");
+        Configuration conf = RevisionManagerConfiguration.create(getHbaseConf());
+        conf.set(RMConstants.ZOOKEEPER_DATADIR, "/rm_base");
         ZKBasedRevisionManager  manager = new ZKBasedRevisionManager();
-        manager.initialize(props);
+        manager.initialize(conf);
         manager.open();
         ZKUtil zkutil = new ZKUtil(sb.toString(), "/rm_base");
 
@@ -132,11 +131,10 @@ public class TestRevisionManager extends
 
         int port = getHbaseConf().getInt("hbase.zookeeper.property.clientPort", 2181);
         String host = getHbaseConf().get("hbase.zookeeper.quorum");
-        Properties props = new Properties();
-        props.put(ZKBasedRevisionManager.HOSTLIST, host + ':' + port);
-        props.put(ZKBasedRevisionManager.DATADIR, "/rm_base");
+        Configuration conf = RevisionManagerConfiguration.create(getHbaseConf());
+        conf.set(RMConstants.ZOOKEEPER_DATADIR, "/rm_base");
         ZKBasedRevisionManager  manager = new ZKBasedRevisionManager();
-        manager.initialize(props);
+        manager.initialize(conf);
         manager.open();
         ZKUtil zkutil = new ZKUtil(host + ':' + port, "/rm_base");
 
@@ -196,11 +194,10 @@ public class TestRevisionManager extends
             sb.append(port);
         }
 
-        Properties props = new Properties();
-        props.put(ZKBasedRevisionManager.HOSTLIST, sb.toString());
-        props.put(ZKBasedRevisionManager.DATADIR, "/rm_base");
+        Configuration conf = RevisionManagerConfiguration.create(getHbaseConf());
+        conf.set(RMConstants.ZOOKEEPER_DATADIR, "/rm_base");
         ZKBasedRevisionManager  manager = new ZKBasedRevisionManager();
-        manager.initialize(props);
+        manager.initialize(conf);
         manager.open();
         String tableName = newTableName("testTable");
         List<String> columnFamilies = Arrays.asList("cf1", "cf2");
@@ -221,11 +218,10 @@ public class TestRevisionManager extends
     public void testCreateSnapshot() throws IOException{
         int port = getHbaseConf().getInt("hbase.zookeeper.property.clientPort", 2181);
         String host = getHbaseConf().get("hbase.zookeeper.quorum");
-        Properties props = new Properties();
-        props.put(ZKBasedRevisionManager.HOSTLIST, host + ':' + port);
-        props.put(ZKBasedRevisionManager.DATADIR, "/rm_base");
+        Configuration conf = RevisionManagerConfiguration.create(getHbaseConf());
+        conf.set(RMConstants.ZOOKEEPER_DATADIR, "/rm_base");
         ZKBasedRevisionManager  manager = new ZKBasedRevisionManager();
-        manager.initialize(props);
+        manager.initialize(conf);
         manager.open();
         String tableName = newTableName("testTable");
         List<String> cfOne = Arrays.asList("cf1", "cf2");

Added: incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManagerConfiguration.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManagerConfiguration.java?rev=1340149&view=auto
==============================================================================
--- incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManagerConfiguration.java (added)
+++ incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManagerConfiguration.java Fri May 18 16:10:03 2012
@@ -0,0 +1,16 @@
+package org.apache.hcatalog.hbase.snapshot;
+
+import org.apache.hadoop.conf.Configuration;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+public class TestRevisionManagerConfiguration {
+
+    @Test
+    public void testDefault() {
+        Configuration conf = RevisionManagerConfiguration.create();
+        assertEquals("org.apache.hcatalog.hbase.snapshot.ZKBasedRevisionManager",
+                     conf.get(RevisionManagerFactory.REVISION_MGR_IMPL_CLASS));
+    }
+}

Modified: incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManagerEndpoint.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManagerEndpoint.java?rev=1340149&r1=1340148&r2=1340149&view=diff
==============================================================================
--- incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManagerEndpoint.java (original)
+++ incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManagerEndpoint.java Fri May 18 16:10:03 2012
@@ -41,7 +41,7 @@ public class TestRevisionManagerEndpoint
     testConf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
         "org.apache.hcatalog.hbase.snapshot.RevisionManagerEndpoint",
         "org.apache.hadoop.hbase.coprocessor.GenericEndpoint");
-    testConf.set(RevisionManagerEndpoint.REVISION_MGR_ENDPOINT_IMPL_CLASS, MockRM.class.getName());
+    testConf.set(RMConstants.REVISION_MGR_ENDPOINT_IMPL_CLASS, MockRM.class.getName());
   }
 
   /**
@@ -106,8 +106,9 @@ public class TestRevisionManagerEndpoint
     }
 
     @Override
-    public void initialize(Properties properties) {
-      INSTANCES.put("default", this);
+    public void initialize(Configuration conf) {
+      if(!INSTANCES.containsKey(DEFAULT_INSTANCE))
+        INSTANCES.put(DEFAULT_INSTANCE, this);
     }
 
     @Override

Modified: incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestZNodeSetUp.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestZNodeSetUp.java?rev=1340149&r1=1340148&r2=1340149&view=diff
==============================================================================
--- incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestZNodeSetUp.java (original)
+++ incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestZNodeSetUp.java Fri May 18 16:10:03 2012
@@ -26,6 +26,7 @@ import java.net.URI;
 import java.util.Map;
 
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hive.cli.CliSessionState;
 import org.apache.hadoop.hive.conf.HiveConf;
@@ -64,7 +65,9 @@ public class TestZNodeSetUp extends Skel
                 hcatConf.set(el.getKey(), el.getValue());
             }
         }
-        hcatConf.set("revision.manager.zk.DataDir", "/rm_base");
+        HBaseConfiguration.merge(hcatConf,
+                RevisionManagerConfiguration.create());
+        hcatConf.set(RMConstants.ZOOKEEPER_DATADIR, "/rm_base");
         SessionState.start(new CliSessionState(hcatConf));
         hcatDriver = new HCatDriver();
 



Mime
View raw message