falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From b...@apache.org
Subject falcon git commit: FALCON-2090 HDFS Snapshot failed with UnknownHostException in HA mode
Date Thu, 21 Jul 2016 18:44:19 GMT
Repository: falcon
Updated Branches:
  refs/heads/master 752511bbe -> 3ed804e92


FALCON-2090 HDFS Snapshot failed with UnknownHostException in HA mode

…uling in HA Mode

Author: bvellanki <bvellanki@hortonworks.com>

Reviewers: "Peeyush B <pbishnoi@hortonworks.com>"

Closes #237 from bvellanki/FALCON-2090


Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/3ed804e9
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/3ed804e9
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/3ed804e9

Branch: refs/heads/master
Commit: 3ed804e92d1434e42a057dc68a6faa6d2fb248ce
Parents: 752511b
Author: bvellanki <bvellanki@hortonworks.com>
Authored: Thu Jul 21 11:44:14 2016 -0700
Committer: bvellanki <bvellanki@hortonworks.com>
Committed: Thu Jul 21 11:44:14 2016 -0700

----------------------------------------------------------------------
 .../replication/HdfsSnapshotReplicator.java          |  8 ++++++--
 .../snapshots/retention/HdfsSnapshotEvictor.java     |  6 ++++--
 .../falcon/snapshots/util/HdfsSnapshotUtil.java      | 15 ++++++++++-----
 .../replication/HdfsSnapshotReplicatorTest.java      | 12 ++++++------
 .../java/org/apache/falcon/entity/ClusterHelper.java |  5 ++---
 5 files changed, 28 insertions(+), 18 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/falcon/blob/3ed804e9/addons/hdfs-snapshot-mirroring/src/main/java/org/apache/falcon/snapshots/replication/HdfsSnapshotReplicator.java
----------------------------------------------------------------------
diff --git a/addons/hdfs-snapshot-mirroring/src/main/java/org/apache/falcon/snapshots/replication/HdfsSnapshotReplicator.java
b/addons/hdfs-snapshot-mirroring/src/main/java/org/apache/falcon/snapshots/replication/HdfsSnapshotReplicator.java
index 2e41cc0..6f5defe 100644
--- a/addons/hdfs-snapshot-mirroring/src/main/java/org/apache/falcon/snapshots/replication/HdfsSnapshotReplicator.java
+++ b/addons/hdfs-snapshot-mirroring/src/main/java/org/apache/falcon/snapshots/replication/HdfsSnapshotReplicator.java
@@ -73,8 +73,12 @@ public class HdfsSnapshotReplicator extends Configured implements Tool
{
         String sourceStorageUrl = cmd.getOptionValue(HdfsSnapshotMirrorProperties.SOURCE_NN.getName());
         String targetStorageUrl = cmd.getOptionValue(HdfsSnapshotMirrorProperties.TARGET_NN.getName());
 
-        DistributedFileSystem sourceFs = HdfsSnapshotUtil.getSourceFileSystem(cmd);
-        DistributedFileSystem targetFs = HdfsSnapshotUtil.getTargetFileSystem(cmd);
+        // Always add to getConf() so that configuration set by oozie action is
+        // available when creating DistributedFileSystem.
+        DistributedFileSystem sourceFs = HdfsSnapshotUtil.getSourceFileSystem(cmd,
+                new Configuration(getConf()));
+        DistributedFileSystem targetFs = HdfsSnapshotUtil.getTargetFileSystem(cmd,
+                new Configuration(getConf()));
 
         String currentSnapshotName = HdfsSnapshotUtil.SNAPSHOT_PREFIX
                 + cmd.getOptionValue(HdfsSnapshotMirrorProperties.SNAPSHOT_JOB_NAME.getName())

http://git-wip-us.apache.org/repos/asf/falcon/blob/3ed804e9/addons/hdfs-snapshot-mirroring/src/main/java/org/apache/falcon/snapshots/retention/HdfsSnapshotEvictor.java
----------------------------------------------------------------------
diff --git a/addons/hdfs-snapshot-mirroring/src/main/java/org/apache/falcon/snapshots/retention/HdfsSnapshotEvictor.java
b/addons/hdfs-snapshot-mirroring/src/main/java/org/apache/falcon/snapshots/retention/HdfsSnapshotEvictor.java
index 22e3377..a50e770 100644
--- a/addons/hdfs-snapshot-mirroring/src/main/java/org/apache/falcon/snapshots/retention/HdfsSnapshotEvictor.java
+++ b/addons/hdfs-snapshot-mirroring/src/main/java/org/apache/falcon/snapshots/retention/HdfsSnapshotEvictor.java
@@ -60,8 +60,10 @@ public class HdfsSnapshotEvictor extends Configured implements Tool {
     @Override
     public int run(String[] args) throws Exception {
         CommandLine cmd = getCommand(args);
-        DistributedFileSystem sourceFs = HdfsSnapshotUtil.getSourceFileSystem(cmd);
-        DistributedFileSystem targetFs = HdfsSnapshotUtil.getTargetFileSystem(cmd);
+        DistributedFileSystem sourceFs = HdfsSnapshotUtil.getSourceFileSystem(cmd,
+                new Configuration(getConf()));
+        DistributedFileSystem targetFs = HdfsSnapshotUtil.getTargetFileSystem(cmd,
+                new Configuration(getConf()));
 
         String sourceDir = cmd.getOptionValue(HdfsSnapshotMirrorProperties.SOURCE_SNAPSHOT_DIR.getName());
         String targetDir = cmd.getOptionValue(HdfsSnapshotMirrorProperties.TARGET_SNAPSHOT_DIR.getName());

http://git-wip-us.apache.org/repos/asf/falcon/blob/3ed804e9/addons/hdfs-snapshot-mirroring/src/main/java/org/apache/falcon/snapshots/util/HdfsSnapshotUtil.java
----------------------------------------------------------------------
diff --git a/addons/hdfs-snapshot-mirroring/src/main/java/org/apache/falcon/snapshots/util/HdfsSnapshotUtil.java
b/addons/hdfs-snapshot-mirroring/src/main/java/org/apache/falcon/snapshots/util/HdfsSnapshotUtil.java
index 5196791..88f6fd9 100644
--- a/addons/hdfs-snapshot-mirroring/src/main/java/org/apache/falcon/snapshots/util/HdfsSnapshotUtil.java
+++ b/addons/hdfs-snapshot-mirroring/src/main/java/org/apache/falcon/snapshots/util/HdfsSnapshotUtil.java
@@ -19,6 +19,7 @@
 package org.apache.falcon.snapshots.util;
 
 import org.apache.commons.cli.CommandLine;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.falcon.FalconException;
 import org.apache.falcon.entity.ClusterHelper;
 import org.apache.falcon.extensions.mirroring.hdfsSnapshot.HdfsSnapshotMirrorProperties;
@@ -37,29 +38,33 @@ public final class HdfsSnapshotUtil {
 
     private HdfsSnapshotUtil() {}
 
-    public static DistributedFileSystem getSourceFileSystem(CommandLine cmd) throws FalconException
{
+    public static DistributedFileSystem getSourceFileSystem(CommandLine cmd,
+                                                            Configuration conf) throws FalconException
{
         String sourceStorageUrl = cmd.getOptionValue(HdfsSnapshotMirrorProperties.SOURCE_NN.getName());
         String sourceExecuteEndpoint = cmd.getOptionValue(HdfsSnapshotMirrorProperties.SOURCE_EXEC_URL.getName());
         String sourcePrincipal = parseKerberosPrincipal(cmd.getOptionValue(
                 HdfsSnapshotMirrorProperties.SOURCE_NN_KERBEROS_PRINCIPAL.getName()));
-        Configuration sourceConf = ClusterHelper.getConfiguration(sourceStorageUrl,
+
+        Configuration sourceConf = ClusterHelper.getConfiguration(conf, sourceStorageUrl,
                 sourceExecuteEndpoint, sourcePrincipal);
         return HadoopClientFactory.get().createDistributedProxiedFileSystem(sourceConf);
     }
 
-    public static DistributedFileSystem getTargetFileSystem(CommandLine cmd) throws FalconException
{
+    public static DistributedFileSystem getTargetFileSystem(CommandLine cmd,
+                                                            Configuration conf) throws FalconException
{
         String targetStorageUrl = cmd.getOptionValue(HdfsSnapshotMirrorProperties.TARGET_NN.getName());
         String taregtExecuteEndpoint = cmd.getOptionValue(HdfsSnapshotMirrorProperties.TARGET_EXEC_URL.getName());
         String targetPrincipal = parseKerberosPrincipal(cmd.getOptionValue(
                 HdfsSnapshotMirrorProperties.TARGET_NN_KERBEROS_PRINCIPAL.getName()));
 
-        Configuration targetConf = ClusterHelper.getConfiguration(targetStorageUrl,
+        Configuration targetConf = ClusterHelper.getConfiguration(conf, targetStorageUrl,
                 taregtExecuteEndpoint, targetPrincipal);
         return HadoopClientFactory.get().createDistributedProxiedFileSystem(targetConf);
     }
 
     public static String parseKerberosPrincipal(String principal) {
-        if (principal.equals(HdfsSnapshotMirroringExtension.EMPTY_KERBEROS_PRINCIPAL)) {
+        if (StringUtils.isEmpty(principal)
+                || principal.equals(HdfsSnapshotMirroringExtension.EMPTY_KERBEROS_PRINCIPAL))
{
             return null;
         }
         return principal;

http://git-wip-us.apache.org/repos/asf/falcon/blob/3ed804e9/addons/hdfs-snapshot-mirroring/src/test/java/org/apache/falcon/snapshots/replication/HdfsSnapshotReplicatorTest.java
----------------------------------------------------------------------
diff --git a/addons/hdfs-snapshot-mirroring/src/test/java/org/apache/falcon/snapshots/replication/HdfsSnapshotReplicatorTest.java
b/addons/hdfs-snapshot-mirroring/src/test/java/org/apache/falcon/snapshots/replication/HdfsSnapshotReplicatorTest.java
index 7924214..fe7ced5 100644
--- a/addons/hdfs-snapshot-mirroring/src/test/java/org/apache/falcon/snapshots/replication/HdfsSnapshotReplicatorTest.java
+++ b/addons/hdfs-snapshot-mirroring/src/test/java/org/apache/falcon/snapshots/replication/HdfsSnapshotReplicatorTest.java
@@ -25,7 +25,7 @@ import org.apache.falcon.entity.store.ConfigurationStore;
 import org.apache.falcon.entity.v0.EntityType;
 import org.apache.falcon.entity.v0.cluster.Cluster;
 import org.apache.falcon.extensions.mirroring.hdfsSnapshot.HdfsSnapshotMirrorProperties;
-import org.apache.falcon.hadoop.HadoopClientFactory;
+import org.apache.falcon.snapshots.util.HdfsSnapshotUtil;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsAction;
@@ -72,6 +72,7 @@ public class HdfsSnapshotReplicatorTest extends HdfsSnapshotReplicator {
 
     @BeforeClass
     public void init() throws Exception {
+        this.setConf(new Configuration());
         baseDir = Files.createTempDirectory("test_snapshot-replication").toFile().getAbsoluteFile();
         miniDFSCluster = MiniHdfsClusterUtil.initMiniDfs(MiniHdfsClusterUtil.SNAPSHOT_REPL_TEST_PORT,
baseDir);
         miniDfs = miniDFSCluster.getFileSystem();
@@ -100,14 +101,13 @@ public class HdfsSnapshotReplicatorTest extends HdfsSnapshotReplicator
{
 
     @Test
     public void replicationTest() throws Exception {
-        Configuration sourceConf = ClusterHelper.getConfiguration(sourceCluster);
-        this.setConf(sourceConf);
-        Configuration targetConf = ClusterHelper.getConfiguration(targetCluster);
         sourceStorageUrl = ClusterHelper.getStorageUrl(sourceCluster);
         targetStorageUrl = ClusterHelper.getStorageUrl(targetCluster);
 
-        DistributedFileSystem sourceFs = HadoopClientFactory.get().createDistributedProxiedFileSystem(sourceConf);
-        DistributedFileSystem targetFs = HadoopClientFactory.get().createDistributedProxiedFileSystem(targetConf);
+        DistributedFileSystem sourceFs = HdfsSnapshotUtil.getSourceFileSystem(cmd,
+                new Configuration(getConf()));
+        DistributedFileSystem targetFs = HdfsSnapshotUtil.getTargetFileSystem(cmd,
+                new Configuration(getConf()));
 
         // create dir1, create snapshot, invoke copy, check file in target, create snapshot
on target
         Path dir1 = new Path(sourceDir, "dir1");

http://git-wip-us.apache.org/repos/asf/falcon/blob/3ed804e9/common/src/main/java/org/apache/falcon/entity/ClusterHelper.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/entity/ClusterHelper.java b/common/src/main/java/org/apache/falcon/entity/ClusterHelper.java
index 9e16fa4..f89def3 100644
--- a/common/src/main/java/org/apache/falcon/entity/ClusterHelper.java
+++ b/common/src/main/java/org/apache/falcon/entity/ClusterHelper.java
@@ -75,9 +75,8 @@ public final class ClusterHelper {
         return conf;
     }
 
-    public static Configuration getConfiguration(String storageUrl, String executeEndPoint,
-                                                 String kerberosPrincipal) {
-        Configuration conf = new Configuration();
+    public static Configuration getConfiguration(Configuration conf, String storageUrl,
+                                                 String executeEndPoint, String kerberosPrincipal)
{
         conf.set(HadoopClientFactory.FS_DEFAULT_NAME_KEY, storageUrl);
         conf.set(HadoopClientFactory.MR_JT_ADDRESS_KEY, executeEndPoint);
         conf.set(HadoopClientFactory.YARN_RM_ADDRESS_KEY, executeEndPoint);


Mime
View raw message