hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ct...@apache.org
Subject hive git commit: HIVE-13527: Using deprecated APIs in HBase client causes zookeeper connection leaks (Naveen Gangam via Chaoyu Tang)
Date Sun, 24 Apr 2016 00:50:39 GMT
Repository: hive
Updated Branches:
  refs/heads/master 287f0451b -> 74197688f


HIVE-13527: Using deprecated APIs in HBase client causes zookeeper connection leaks (Naveen
Gangam via Chaoyu Tang)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/74197688
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/74197688
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/74197688

Branch: refs/heads/master
Commit: 74197688f18d9a6bee6393cfe1d7ab635beb3db5
Parents: 287f045
Author: ctang <ctang@cloudera.com>
Authored: Sat Apr 23 20:50:00 2016 -0400
Committer: ctang <ctang@cloudera.com>
Committed: Sat Apr 23 20:50:00 2016 -0400

----------------------------------------------------------------------
 .../hive/hbase/HiveHBaseTableInputFormat.java   | 33 ++++++++++++++++++--
 1 file changed, 31 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/74197688/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java
----------------------------------------------------------------------
diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java
b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java
index 88d1865..d9db624 100644
--- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java
+++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java
@@ -26,6 +26,9 @@ import java.util.Map;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.Scan;
@@ -84,6 +87,7 @@ public class HiveHBaseTableInputFormat extends TableInputFormatBase
 
   static final Logger LOG = LoggerFactory.getLogger(HiveHBaseTableInputFormat.class);
   private static final Object hbaseTableMonitor = new Object();
+  private Connection conn = null;
 
   @Override
   public RecordReader<ImmutableBytesWritable, ResultWritable> getRecordReader(
@@ -94,7 +98,11 @@ public class HiveHBaseTableInputFormat extends TableInputFormatBase
     HBaseSplit hbaseSplit = (HBaseSplit) split;
     TableSplit tableSplit = hbaseSplit.getTableSplit();
 
-    setHTable(HiveHBaseInputFormatUtil.getTable(jobConf));
+    if (conn == null) {
+      conn = ConnectionFactory.createConnection(HBaseConfiguration.create(jobConf));
+    }
+    initializeTable(conn, tableSplit.getTable());
+
     setScan(HiveHBaseInputFormatUtil.getScan(jobConf));
 
     Job job = new Job(jobConf);
@@ -107,6 +115,10 @@ public class HiveHBaseTableInputFormat extends TableInputFormatBase
       recordReader.initialize(tableSplit, tac);
     } catch (InterruptedException e) {
       closeTable(); // Free up the HTable connections
+      if (conn != null) {
+        conn.close();
+        conn = null;
+      }
       throw new IOException("Failed to initialize RecordReader", e);
     }
 
@@ -116,6 +128,10 @@ public class HiveHBaseTableInputFormat extends TableInputFormatBase
       public void close() throws IOException {
         recordReader.close();
         closeTable();
+        if (conn != null) {
+          conn.close();
+          conn = null;
+        }
       }
 
       @Override
@@ -442,7 +458,12 @@ public class HiveHBaseTableInputFormat extends TableInputFormatBase
     }
 
     String hbaseTableName = jobConf.get(HBaseSerDe.HBASE_TABLE_NAME);
-    setHTable(new HTable(HBaseConfiguration.create(jobConf), Bytes.toBytes(hbaseTableName)));
+    if (conn == null) {
+      conn = ConnectionFactory.createConnection(HBaseConfiguration.create(jobConf));
+    }
+    TableName tableName = TableName.valueOf(hbaseTableName);
+    initializeTable(conn, tableName);
+
     String hbaseColumnsMapping = jobConf.get(HBaseSerDe.HBASE_COLUMNS_MAPPING);
     boolean doColumnRegexMatching = jobConf.getBoolean(HBaseSerDe.HBASE_COLUMNS_REGEX_MATCHING,
true);
 
@@ -509,6 +530,10 @@ public class HiveHBaseTableInputFormat extends TableInputFormatBase
       return results;
     } finally {
       closeTable();
+      if (conn != null) {
+        conn.close();
+        conn = null;
+      }
     }
   }
 
@@ -516,6 +541,10 @@ public class HiveHBaseTableInputFormat extends TableInputFormatBase
   protected void finalize() throws Throwable {
     try {
       closeTable();
+      if (conn != null) {
+        conn.close();
+        conn = null;
+      }
     } finally {
       super.finalize();
     }


Mime
View raw message