falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From b...@apache.org
Subject falcon git commit: FALCON-1941 HiveDR fails with NN-HA enabled on both the source and target clusters
Date Tue, 10 May 2016 21:50:07 GMT
Repository: falcon
Updated Branches:
  refs/heads/master 1b8a93307 -> 8bd7a5c87


FALCON-1941 HiveDR fails with NN-HA enabled on both the source and target clusters

Fix for HiveDR to work NN-HA enabled clusters

Author: Venkat Ranganathan <venkat@hortonworks.com>

Reviewers: "Ying Zheng <yzheng@hortonworks.com>, Balu Vellanki <balu@apache.org>"

Closes #133 from vrangan/FALCON-1941


Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/8bd7a5c8
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/8bd7a5c8
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/8bd7a5c8

Branch: refs/heads/master
Commit: 8bd7a5c87f9c80119471b1b29a72067e6650c20e
Parents: 1b8a933
Author: Venkat Ranganathan <venkat@hortonworks.com>
Authored: Tue May 10 14:50:02 2016 -0700
Committer: bvellanki <bvellanki@hortonworks.com>
Committed: Tue May 10 14:50:02 2016 -0700

----------------------------------------------------------------------
 .../org/apache/falcon/hive/util/FileUtils.java  |  4 +--
 .../apache/falcon/hive/util/HiveDRUtils.java    | 34 +++++++++++++++++---
 2 files changed, 31 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/falcon/blob/8bd7a5c8/addons/hivedr/src/main/java/org/apache/falcon/hive/util/FileUtils.java
----------------------------------------------------------------------
diff --git a/addons/hivedr/src/main/java/org/apache/falcon/hive/util/FileUtils.java b/addons/hivedr/src/main/java/org/apache/falcon/hive/util/FileUtils.java
index 6bd6319..483a79c 100644
--- a/addons/hivedr/src/main/java/org/apache/falcon/hive/util/FileUtils.java
+++ b/addons/hivedr/src/main/java/org/apache/falcon/hive/util/FileUtils.java
@@ -40,8 +40,8 @@ public final class FileUtils {
 
     private FileUtils() {}
 
-    public static Configuration getConfiguration(final String writeEP, final String nnKerberosPrincipal)
{
-        Configuration conf = new Configuration();
+    public static Configuration getConfiguration(final String writeEP, final String nnKerberosPrincipal)
throws IOException {
+        Configuration conf = HiveDRUtils.getDefaultConf();
         conf.set("fs.defaultFS", writeEP);
         if (StringUtils.isNotEmpty(nnKerberosPrincipal)) {
             conf.set("dfs.namenode.kerberos.principal", nnKerberosPrincipal);

http://git-wip-us.apache.org/repos/asf/falcon/blob/8bd7a5c8/addons/hivedr/src/main/java/org/apache/falcon/hive/util/HiveDRUtils.java
----------------------------------------------------------------------
diff --git a/addons/hivedr/src/main/java/org/apache/falcon/hive/util/HiveDRUtils.java b/addons/hivedr/src/main/java/org/apache/falcon/hive/util/HiveDRUtils.java
index dff0803..ddcc178 100644
--- a/addons/hivedr/src/main/java/org/apache/falcon/hive/util/HiveDRUtils.java
+++ b/addons/hivedr/src/main/java/org/apache/falcon/hive/util/HiveDRUtils.java
@@ -25,6 +25,8 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.util.Shell;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.File;
 import java.io.IOException;
@@ -56,6 +58,8 @@ public final class HiveDRUtils {
 
     public static final String SEPARATOR = File.separator;
 
+    private static final Logger LOG = LoggerFactory.getLogger(HiveDRUtils.class);
+
     private HiveDRUtils() {}
 
     public static ReplicationType getReplicationType(List<String> sourceTables) {
@@ -65,12 +69,32 @@ public final class HiveDRUtils {
 
     public static Configuration getDefaultConf() throws IOException {
         Configuration conf = new Configuration();
-        conf.addResource(new Path("file:///", System.getProperty("oozie.action.conf.xml")));
-        String delegationToken = getFilePathFromEnv("HADOOP_TOKEN_FILE_LOCATION");
-        if (delegationToken != null) {
-            conf.set("mapreduce.job.credentials.binary", delegationToken);
-            conf.set("tez.credentials.path", delegationToken);
+        Path confPath = new Path("file:///", System.getProperty("oozie.action.conf.xml"));
+
+        final boolean actionConfExists = confPath.getFileSystem(conf).exists(confPath);
+        LOG.info("Oozie Action conf {} found ? {}", confPath, actionConfExists);
+        if (actionConfExists) {
+            LOG.info("Oozie Action conf found, adding path={}, conf={}", confPath, conf.toString());
+            conf.addResource(confPath);
         }
+
+        String tokenFile = System.getenv("HADOOP_TOKEN_FILE_LOCATION");
+        if (StringUtils.isNotBlank(tokenFile)) {
+            if (Shell.WINDOWS) {
+                if (tokenFile.charAt(0) == '"') {
+                    tokenFile = tokenFile.substring(1);
+                }
+                if (tokenFile.charAt(tokenFile.length() - 1) == '"') {
+                    tokenFile = tokenFile.substring(0, tokenFile.length() - 1);
+                }
+            }
+
+            conf.set("mapreduce.job.credentials.binary", tokenFile);
+            System.setProperty("mapreduce.job.credentials.binary", tokenFile);
+            conf.set("tez.credentials.path", tokenFile);
+            System.setProperty("tez.credentials.path", tokenFile);
+        }
+
         return conf;
     }
 


Mime
View raw message