falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From b...@apache.org
Subject falcon git commit: FALCON-2095 HiveDR fails in NN-HA config
Date Wed, 27 Jul 2016 20:22:53 GMT
Repository: falcon
Updated Branches:
  refs/heads/master c588628aa -> ea0492dac


FALCON-2095 HiveDR fails in NN-HA config

Fix validated in a two cluster configuration with NN/RM HA setup

Author: Venkat Ranganathan <venkat@hortonworks.com>

Reviewers: "Balu Vellanki <balu@apache.org>"

Closes #240 from vrangan/FALCON-2095


Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/ea0492da
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/ea0492da
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/ea0492da

Branch: refs/heads/master
Commit: ea0492dacd3d0929751060bee26b1e5a61178431
Parents: c588628
Author: Venkat Ranganathan <venkat@hortonworks.com>
Authored: Wed Jul 27 13:22:40 2016 -0700
Committer: bvellanki <bvellanki@hortonworks.com>
Committed: Wed Jul 27 13:22:40 2016 -0700

----------------------------------------------------------------------
 .../main/java/org/apache/falcon/hive/HiveDRTool.java    |  6 +++---
 .../org/apache/falcon/hive/mapreduce/CopyReducer.java   |  3 ++-
 .../java/org/apache/falcon/hive/util/EventUtils.java    |  6 +++---
 .../java/org/apache/falcon/hive/util/FileUtils.java     | 12 ++++++------
 4 files changed, 14 insertions(+), 13 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/falcon/blob/ea0492da/addons/hivedr/src/main/java/org/apache/falcon/hive/HiveDRTool.java
----------------------------------------------------------------------
diff --git a/addons/hivedr/src/main/java/org/apache/falcon/hive/HiveDRTool.java b/addons/hivedr/src/main/java/org/apache/falcon/hive/HiveDRTool.java
index e45b0d8..2701e02 100644
--- a/addons/hivedr/src/main/java/org/apache/falcon/hive/HiveDRTool.java
+++ b/addons/hivedr/src/main/java/org/apache/falcon/hive/HiveDRTool.java
@@ -143,13 +143,13 @@ public class HiveDRTool extends Configured implements Tool {
         LOG.info("srcStaginPath: {}", inputOptions.getSourceStagingPath());
         LOG.info("tgtStaginPath: {}", inputOptions.getTargetStagingPath());
 
-        Configuration sourceConf = FileUtils.getConfiguration(inputOptions.getSourceWriteEP(),
+        Configuration sourceConf = FileUtils.getConfiguration(getConf(), inputOptions.getSourceWriteEP(),
                 inputOptions.getSourceNNKerberosPrincipal());
         sourceClusterFS = FileSystem.get(sourceConf);
-        Configuration targetConf = FileUtils.getConfiguration(inputOptions.getTargetWriteEP(),
+        Configuration targetConf = FileUtils.getConfiguration(getConf(), inputOptions.getTargetWriteEP(),
                 inputOptions.getTargetNNKerberosPrincipal());
         targetClusterFS = FileSystem.get(targetConf);
-        jobConf = FileUtils.getConfiguration(inputOptions.getJobClusterWriteEP(),
+        jobConf = FileUtils.getConfiguration(getConf(), inputOptions.getJobClusterWriteEP(),
                 inputOptions.getJobClusterNNPrincipal());
         jobFS = FileSystem.get(jobConf);
 

http://git-wip-us.apache.org/repos/asf/falcon/blob/ea0492da/addons/hivedr/src/main/java/org/apache/falcon/hive/mapreduce/CopyReducer.java
----------------------------------------------------------------------
diff --git a/addons/hivedr/src/main/java/org/apache/falcon/hive/mapreduce/CopyReducer.java
b/addons/hivedr/src/main/java/org/apache/falcon/hive/mapreduce/CopyReducer.java
index f4bb31c..7c415c3 100644
--- a/addons/hivedr/src/main/java/org/apache/falcon/hive/mapreduce/CopyReducer.java
+++ b/addons/hivedr/src/main/java/org/apache/falcon/hive/mapreduce/CopyReducer.java
@@ -48,7 +48,8 @@ public class CopyReducer extends Reducer<Text, Text, Text, Text> {
     @Override
     protected void setup(Context context) throws IOException, InterruptedException {
         Configuration conf = context.getConfiguration();
-        FileSystem fs= FileSystem.get(FileUtils.getConfiguration(
+        FileSystem fs= FileSystem.get(
+                FileUtils.getConfiguration(context.getConfiguration(),
                 conf.get(HiveDRArgs.TARGET_NN.getName()),
                 conf.get(HiveDRArgs.TARGET_NN_KERBEROS_PRINCIPAL.getName())));
         hiveDRStore = new HiveDRStatusStore(fs);

http://git-wip-us.apache.org/repos/asf/falcon/blob/ea0492da/addons/hivedr/src/main/java/org/apache/falcon/hive/util/EventUtils.java
----------------------------------------------------------------------
diff --git a/addons/hivedr/src/main/java/org/apache/falcon/hive/util/EventUtils.java b/addons/hivedr/src/main/java/org/apache/falcon/hive/util/EventUtils.java
index 590a7e3..05b5f96 100644
--- a/addons/hivedr/src/main/java/org/apache/falcon/hive/util/EventUtils.java
+++ b/addons/hivedr/src/main/java/org/apache/falcon/hive/util/EventUtils.java
@@ -142,9 +142,9 @@ public class EventUtils {
         LOG.info("Initializing staging directory");
         sourceStagingUri = new Path(sourceNN, sourceStagingPath).toString();
         targetStagingUri = new Path(targetNN, targetStagingPath).toString();
-        sourceFileSystem = FileSystem.get(FileUtils.getConfiguration(sourceNN, sourceNNKerberosPrincipal));
-        jobFileSystem = FileSystem.get(FileUtils.getConfiguration(jobNN, jobNNKerberosPrincipal));
-        targetFileSystem = FileSystem.get(FileUtils.getConfiguration(targetNN, targetNNKerberosPrincipal));
+        sourceFileSystem = FileSystem.get(FileUtils.getConfiguration(conf, sourceNN, sourceNNKerberosPrincipal));
+        jobFileSystem = FileSystem.get(FileUtils.getConfiguration(conf, jobNN, jobNNKerberosPrincipal));
+        targetFileSystem = FileSystem.get(FileUtils.getConfiguration(conf, targetNN, targetNNKerberosPrincipal));
     }
 
     private String readEvents(Path eventFileName) throws IOException {

http://git-wip-us.apache.org/repos/asf/falcon/blob/ea0492da/addons/hivedr/src/main/java/org/apache/falcon/hive/util/FileUtils.java
----------------------------------------------------------------------
diff --git a/addons/hivedr/src/main/java/org/apache/falcon/hive/util/FileUtils.java b/addons/hivedr/src/main/java/org/apache/falcon/hive/util/FileUtils.java
index 8b5c865..dae4849 100644
--- a/addons/hivedr/src/main/java/org/apache/falcon/hive/util/FileUtils.java
+++ b/addons/hivedr/src/main/java/org/apache/falcon/hive/util/FileUtils.java
@@ -42,14 +42,14 @@ public final class FileUtils {
 
     private FileUtils() {}
 
-    public static Configuration getConfiguration(final String writeEP,
-                                                 final String nnKerberosPrincipal) throws
IOException {
-        Configuration conf = HiveDRUtils.getDefaultConf();
-        conf.set("fs.defaultFS", writeEP);
+    public static Configuration getConfiguration(Configuration conf,
+                                final String writeEP, final String nnKerberosPrincipal) throws
IOException {
+        Configuration newConf = new Configuration(conf);
+        newConf.set("fs.defaultFS", writeEP);
         if (StringUtils.isNotEmpty(nnKerberosPrincipal)) {
-            conf.set("dfs.namenode.kerberos.principal", nnKerberosPrincipal);
+            newConf.set("dfs.namenode.kerberos.principal", nnKerberosPrincipal);
         }
-        return conf;
+        return newConf;
     }
 
     public static void validatePath(final FileSystem fileSystem, final Path basePath) throws
IOException {


Mime
View raw message