pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dvrya...@apache.org
Subject svn commit: r1165138 - in /pig/trunk: CHANGES.txt src/org/apache/pig/backend/hadoop/hbase/HBaseStorage.java
Date Mon, 05 Sep 2011 00:15:17 GMT
Author: dvryaboy
Date: Mon Sep  5 00:15:16 2011
New Revision: 1165138

URL: http://svn.apache.org/viewvc?rev=1165138&view=rev
Log:
PIG-2115: Fix Pig HBaseStorage configuration and setup issues

Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/src/org/apache/pig/backend/hadoop/hbase/HBaseStorage.java

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1165138&r1=1165137&r2=1165138&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Mon Sep  5 00:15:16 2011
@@ -132,6 +132,8 @@ PIG-2011: Speed up TestTypedMap.java (dv
 
 BUG FIXES
 
+PIG-2115: Fix Pig HBaseStorage configuration and setup issues (gbowyer@fastmail.co.uk via
dvryaboy)
+
 PIG-2193: Using HBaseStorage to scan 2 tables in the same Map job produces bad data (rangadi
via dvryaboy)
 
 PIG-2232: "declare" document contains a typo (daijy)

Modified: pig/trunk/src/org/apache/pig/backend/hadoop/hbase/HBaseStorage.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/hbase/HBaseStorage.java?rev=1165138&r1=1165137&r2=1165138&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/hbase/HBaseStorage.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/hbase/HBaseStorage.java Mon Sep  5 00:15:16
2011
@@ -77,6 +77,7 @@ import org.apache.pig.ResourceSchema.Res
 import org.apache.pig.StoreFuncInterface;
 import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigSplit;
 import org.apache.pig.backend.hadoop.hbase.HBaseTableInputFormat.HBaseTableIFBuilder;
+import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
 import org.apache.pig.builtin.Utf8StorageConverter;
 import org.apache.pig.data.DataBag;
 import org.apache.pig.data.DataByteArray;
@@ -530,14 +531,7 @@ public class HBaseStorage extends LoadFu
     @Override
     public void setLocation(String location, Job job) throws IOException {
         job.getConfiguration().setBoolean("pig.noSplitCombination", true);
-        m_conf = job.getConfiguration();
-        HBaseConfiguration.addHbaseResources(m_conf);
-
-        // Make sure the HBase, ZooKeeper, and Guava jars get shipped.
-        TableMapReduceUtil.addDependencyJars(job.getConfiguration(), 
-            org.apache.hadoop.hbase.client.HTable.class,
-            com.google.common.collect.Lists.class,
-            org.apache.zookeeper.ZooKeeper.class);
+        m_conf = initialiseHBaseClassLoaderResources(job);
 
         String tablename = location;
         if (location.startsWith("hbase://")){
@@ -574,6 +568,24 @@ public class HBaseStorage extends LoadFu
         m_conf.set(TableInputFormat.SCAN, convertScanToString(scan));
     }
 
+    private Configuration initialiseHBaseClassLoaderResources(Job job) throws IOException
{
+        Configuration hbaseConfig = initialiseHBaseConfig(job.getConfiguration());
+
+        // Make sure the HBase, ZooKeeper, and Guava jars get shipped.
+        TableMapReduceUtil.addDependencyJars(job.getConfiguration(),
+            org.apache.hadoop.hbase.client.HTable.class,
+            com.google.common.collect.Lists.class,
+            org.apache.zookeeper.ZooKeeper.class);
+
+        return hbaseConfig;
+    }
+
+    private Configuration initialiseHBaseConfig(Configuration conf) {
+        Configuration hbaseConfig = HBaseConfiguration.create();
+        ConfigurationUtil.mergeConf(hbaseConfig, conf);
+        return hbaseConfig;
+    }
+
     @Override
     public String relativeToAbsolutePath(String location, Path curDir)
     throws IOException {
@@ -610,7 +622,7 @@ public class HBaseStorage extends LoadFu
     public OutputFormat getOutputFormat() throws IOException {
         if (outputFormat == null) {
             this.outputFormat = new TableOutputFormat();
-            HBaseConfiguration.addHbaseResources(m_conf);
+            m_conf = initialiseHBaseConfig(m_conf);
             this.outputFormat.setConf(m_conf);            
         }
         return outputFormat;
@@ -751,7 +763,7 @@ public class HBaseStorage extends LoadFu
             schema_ = (ResourceSchema) ObjectSerializer.deserialize(serializedSchema);
         }
 
-        m_conf = HBaseConfiguration.addHbaseResources(job.getConfiguration());
+        m_conf = initialiseHBaseClassLoaderResources(job);
     }
 
     @Override



Mime
View raw message