phoenix-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sama...@apache.org
Subject [16/36] phoenix git commit: PHOENIX-3490 PhoenixStorageHandler doesn't handler hbase configuration properly and don't work in secured env
Date Tue, 22 Nov 2016 02:54:20 GMT
PHOENIX-3490 PhoenixStorageHandler doesn't handler hbase configuration properly and don't work
in secured env


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/3f1be1a9
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/3f1be1a9
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/3f1be1a9

Branch: refs/heads/encodecolumns2
Commit: 3f1be1a9cbab3647d3e915d4935855d68180b808
Parents: b62ebe0
Author: Sergey Soldatov <ssa@apache.org>
Authored: Wed Nov 16 17:07:08 2016 -0800
Committer: Sergey Soldatov <ssa@apache.org>
Committed: Thu Nov 17 12:07:02 2016 -0800

----------------------------------------------------------------------
 .../phoenix/hive/PhoenixStorageHandler.java     | 55 ++++++++++++++++++++
 1 file changed, 55 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/3f1be1a9/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixStorageHandler.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixStorageHandler.java
b/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixStorageHandler.java
index bda2282..a425b7c 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixStorageHandler.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixStorageHandler.java
@@ -19,7 +19,10 @@ package org.apache.phoenix.hive;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.mapred.TableMapReduceUtil;
 import org.apache.hadoop.hive.common.JavaUtils;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.HiveMetaHook;
@@ -34,6 +37,7 @@ import org.apache.hadoop.hive.ql.plan.TableDesc;
 import org.apache.hadoop.hive.ql.session.SessionState;
 import org.apache.hadoop.hive.serde2.Deserializer;
 import org.apache.hadoop.hive.serde2.SerDe;
+import org.apache.hadoop.hive.shims.ShimLoader;
 import org.apache.hadoop.mapred.InputFormat;
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.OutputFormat;
@@ -43,6 +47,7 @@ import org.apache.phoenix.hive.mapreduce.PhoenixOutputFormat;
 import org.apache.phoenix.hive.ppd.PhoenixPredicateDecomposer;
 import org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil;
 
+import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
@@ -54,6 +59,22 @@ import java.util.Properties;
 public class PhoenixStorageHandler extends DefaultStorageHandler implements
         HiveStoragePredicateHandler, InputEstimator {
 
+
+    private Configuration jobConf;
+    private Configuration hbaseConf;
+
+
+    @Override
+    public void setConf(Configuration conf) {
+        jobConf = conf;
+        hbaseConf = HBaseConfiguration.create(conf);
+    }
+
+    @Override
+    public Configuration getConf() {
+        return hbaseConf;
+    }
+
     private static final Log LOG = LogFactory.getLog(PhoenixStorageHandler.class);
 
     public PhoenixStorageHandler() {
@@ -67,6 +88,22 @@ public class PhoenixStorageHandler extends DefaultStorageHandler implements
         return new PhoenixMetaHook();
     }
 
+    @Override
+    public void configureJobConf(TableDesc tableDesc, JobConf jobConf) {
+        try {
+            TableMapReduceUtil.addDependencyJars(jobConf);
+            org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.addDependencyJars(jobConf,
+                    PhoenixStorageHandler.class);
+            JobConf hbaseJobConf = new JobConf(getConf());
+            org.apache.hadoop.hbase.mapred.TableMapReduceUtil.initCredentials(hbaseJobConf);
+            ShimLoader.getHadoopShims().mergeCredentials(jobConf, hbaseJobConf);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+
+
+    }
+
     @SuppressWarnings("rawtypes")
     @Override
     public Class<? extends OutputFormat> getOutputFormatClass() {
@@ -167,6 +204,24 @@ public class PhoenixStorageHandler extends DefaultStorageHandler implements
                 (PhoenixStorageHandlerConstants.ZOOKEEPER_PORT));
         jobProperties.put(HConstants.ZOOKEEPER_ZNODE_PARENT, jobProperties.get
                 (PhoenixStorageHandlerConstants.ZOOKEEPER_PARENT));
+        addHBaseResources(jobConf, jobProperties);
+    }
+
+    /**
+     * Utility method to add hbase-default.xml and hbase-site.xml properties to a new map
+     * if they are not already present in the jobConf.
+     * @param jobConf Job configuration
+     * @param newJobProperties  Map to which new properties should be added
+     */
+    private void addHBaseResources(Configuration jobConf,
+                                   Map<String, String> newJobProperties) {
+        Configuration conf = new Configuration(false);
+        HBaseConfiguration.addHbaseResources(conf);
+        for (Map.Entry<String, String> entry : conf) {
+            if (jobConf.get(entry.getKey()) == null) {
+                newJobProperties.put(entry.getKey(), entry.getValue());
+            }
+        }
     }
 
     @Override


Mime
View raw message