falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From venkat...@apache.org
Subject [2/2] git commit: FALCON-445 Propagate hive credentials defined in cluster entity to hive-site.xml. Contributed by Sowmya Ramesh
Date Fri, 23 May 2014 18:31:15 GMT
FALCON-445 Propagate hive credentials defined in cluster entity to hive-site.xml. Contributed
by Sowmya Ramesh


Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/3d3bde55
Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/3d3bde55
Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/3d3bde55

Branch: refs/heads/master
Commit: 3d3bde558027af1f6ceb914b396cf6a41682625e
Parents: b7afe36
Author: Venkatesh Seetharam <venkatesh@apache.org>
Authored: Fri May 23 11:31:03 2014 -0700
Committer: Venkatesh Seetharam <venkatesh@apache.org>
Committed: Fri May 23 11:31:03 2014 -0700

----------------------------------------------------------------------
 CHANGES.txt                                     |  5 ++++-
 .../org/apache/falcon/entity/ClusterHelper.java | 20 ++++++++++++++++++++
 .../falcon/workflow/OozieWorkflowBuilder.java   |  6 ++++++
 .../OozieProcessWorkflowBuilderTest.java        |  3 ++-
 .../resources/config/cluster/cluster-0.1.xml    |  1 +
 5 files changed, 33 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/3d3bde55/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 0e26c7d..e06fc80 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,7 +7,10 @@ Trunk (Unreleased)
   NEW FEATURES
 
   IMPROVEMENTS
-   FALCON-133 Upgrade to slf4j 1.7.5 and use SLF4J logger. (Jean-Baptiste Onofré 
+   FALCON-445 Propagate hive credentials defined in cluster entity to
+   hive-site.xml (Sowmya Ramesh via Venkatesh Seetharam)
+
+   FALCON-133 Upgrade to slf4j 1.7.5 and use SLF4J logger. (Jean-Baptiste Onofré
    via Shwetha GS)
 
   OPTIMIZATIONS

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/3d3bde55/common/src/main/java/org/apache/falcon/entity/ClusterHelper.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/entity/ClusterHelper.java b/common/src/main/java/org/apache/falcon/entity/ClusterHelper.java
index 3bf9d95..5284d68 100644
--- a/common/src/main/java/org/apache/falcon/entity/ClusterHelper.java
+++ b/common/src/main/java/org/apache/falcon/entity/ClusterHelper.java
@@ -27,6 +27,10 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
 /**
  * Helper to get end points relating to the cluster.
  */
@@ -135,4 +139,20 @@ public final class ClusterHelper {
         }
         return null;
     }
+
+    public static Map<String, String> geHiveProperties(Cluster cluster) {
+        if (cluster.getProperties() != null) {
+            List<Property> properties = cluster.getProperties().getProperties();
+            if (properties != null && !properties.isEmpty()) {
+                Map<String, String> hiveProperties = new HashMap<String, String>();
+                for (Property prop : properties) {
+                    if (prop.getName().startsWith("hive.")) {
+                        hiveProperties.put(prop.getName(), prop.getValue());
+                    }
+                }
+                return hiveProperties;
+            }
+        }
+        return null;
+    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/3d3bde55/oozie/src/main/java/org/apache/falcon/workflow/OozieWorkflowBuilder.java
----------------------------------------------------------------------
diff --git a/oozie/src/main/java/org/apache/falcon/workflow/OozieWorkflowBuilder.java b/oozie/src/main/java/org/apache/falcon/workflow/OozieWorkflowBuilder.java
index f5ff27a..7d84938 100644
--- a/oozie/src/main/java/org/apache/falcon/workflow/OozieWorkflowBuilder.java
+++ b/oozie/src/main/java/org/apache/falcon/workflow/OozieWorkflowBuilder.java
@@ -483,6 +483,12 @@ public abstract class OozieWorkflowBuilder<T extends Entity> extends
WorkflowBui
                     "Registry interface is not defined in cluster: " + cluster.getName());
         }
 
+        // Propagate the hive properties from cluster entity
+        Map<String, String> hiveProperties = ClusterHelper.geHiveProperties(cluster);
+        if (hiveProperties != null && !hiveProperties.isEmpty()) {
+            hiveCredentials.putAll(hiveProperties);
+        }
+
         hiveCredentials.put(METASTOREURIS, metaStoreUrl);
         hiveCredentials.put("hive.metastore.execute.setugi", "true");
         hiveCredentials.put("hcatNode", metaStoreUrl.replace("thrift", "hcat"));

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/3d3bde55/process/src/test/java/org/apache/falcon/converter/OozieProcessWorkflowBuilderTest.java
----------------------------------------------------------------------
diff --git a/process/src/test/java/org/apache/falcon/converter/OozieProcessWorkflowBuilderTest.java
b/process/src/test/java/org/apache/falcon/converter/OozieProcessWorkflowBuilderTest.java
index 8cfa9fc..2522ca3 100644
--- a/process/src/test/java/org/apache/falcon/converter/OozieProcessWorkflowBuilderTest.java
+++ b/process/src/test/java/org/apache/falcon/converter/OozieProcessWorkflowBuilderTest.java
@@ -289,8 +289,9 @@ public class OozieProcessWorkflowBuilderTest extends AbstractTestBase
{
             props.put(prop.getName(), prop.getValue());
         }
 
-        // verify table props
+        // verify table and hive props
         Map<String, String> expected = getExpectedProperties(inFeed, outFeed, process);
+        expected.putAll(ClusterHelper.geHiveProperties(cluster));
         for (Map.Entry<String, String> entry : props.entrySet()) {
             if (expected.containsKey(entry.getKey())) {
                 Assert.assertEquals(entry.getValue(), expected.get(entry.getKey()));

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/3d3bde55/process/src/test/resources/config/cluster/cluster-0.1.xml
----------------------------------------------------------------------
diff --git a/process/src/test/resources/config/cluster/cluster-0.1.xml b/process/src/test/resources/config/cluster/cluster-0.1.xml
index 3780b3f..032cc77 100644
--- a/process/src/test/resources/config/cluster/cluster-0.1.xml
+++ b/process/src/test/resources/config/cluster/cluster-0.1.xml
@@ -39,5 +39,6 @@
     <properties>
         <property name="field1" value="value1"/>
         <property name="field2" value="value2"/>
+        <property name="hive.metastore.client.socket.timeout" value="20"/>
     </properties>
 </cluster>


Mime
View raw message