atlas-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From shweth...@apache.org
Subject incubator-atlas git commit: ATLAS-837 Enhance Sqoop addon to handle export operation (venkatnrangan via shwethags)
Date Mon, 13 Jun 2016 17:04:35 GMT
Repository: incubator-atlas
Updated Branches:
  refs/heads/master 67a1133a3 -> 5b627b5a1


ATLAS-837 Enhance Sqoop addon to handle export operation (venkatnrangan via shwethags)


Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/5b627b5a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/5b627b5a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/5b627b5a

Branch: refs/heads/master
Commit: 5b627b5a1e3103f750fd4ba981ef569ad2377bd1
Parents: 67a1133
Author: Shwetha GS <sshivalingamurthy@hortonworks.com>
Authored: Mon Jun 13 22:34:28 2016 +0530
Committer: Shwetha GS <sshivalingamurthy@hortonworks.com>
Committed: Mon Jun 13 22:34:28 2016 +0530

----------------------------------------------------------------------
 .../org/apache/atlas/sqoop/hook/SqoopHook.java  | 20 +++++++++++++++-----
 .../apache/atlas/sqoop/hook/SqoopHookIT.java    | 15 +++++++++++++++
 release-log.txt                                 |  1 +
 3 files changed, 31 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/5b627b5a/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/hook/SqoopHook.java
----------------------------------------------------------------------
diff --git a/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/hook/SqoopHook.java
b/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/hook/SqoopHook.java
index 18474ad..626c5a7 100644
--- a/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/hook/SqoopHook.java
+++ b/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/hook/SqoopHook.java
@@ -109,8 +109,13 @@ public class SqoopHook extends SqoopJobDataPublisher {
         procRef.set(SqoopDataModelGenerator.NAME, sqoopProcessName);
         procRef.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, sqoopProcessName);
         procRef.set(SqoopDataModelGenerator.OPERATION, data.getOperation());
-        procRef.set(SqoopDataModelGenerator.INPUTS, dbStoreRef);
-        procRef.set(SqoopDataModelGenerator.OUTPUTS, hiveTableRef);
+        if (isImportOperation(data)) {
+            procRef.set(SqoopDataModelGenerator.INPUTS, dbStoreRef);
+            procRef.set(SqoopDataModelGenerator.OUTPUTS, hiveTableRef);
+        } else {
+            procRef.set(SqoopDataModelGenerator.INPUTS, hiveTableRef);
+            procRef.set(SqoopDataModelGenerator.OUTPUTS, dbStoreRef);
+        }
         procRef.set(SqoopDataModelGenerator.USER, data.getUser());
         procRef.set(SqoopDataModelGenerator.START_TIME, new Date(data.getStartTime()));
         procRef.set(SqoopDataModelGenerator.END_TIME, new Date(data.getEndTime()));
@@ -126,15 +131,16 @@ public class SqoopHook extends SqoopJobDataPublisher {
     }
 
     static String getSqoopProcessName(Data data, String clusterName) {
-        StringBuilder name = new StringBuilder(String.format("sqoop import --connect %s",
data.getUrl()));
+        StringBuilder name = new StringBuilder(String.format("sqoop %s --connect %s", data.getOperation(),
+                data.getUrl()));
         if (StringUtils.isNotEmpty(data.getStoreTable())) {
             name.append(" --table ").append(data.getStoreTable());
         }
         if (StringUtils.isNotEmpty(data.getStoreQuery())) {
             name.append(" --query ").append(data.getStoreQuery());
         }
-        name.append(String.format(" --hive-import --hive-database %s --hive-table %s --hive-cluster
%s",
-                data.getHiveDB().toLowerCase(), data.getHiveTable().toLowerCase(), clusterName));
+        name.append(String.format(" --hive-%s --hive-database %s --hive-table %s --hive-cluster
%s",
+                data.getOperation(), data.getHiveDB().toLowerCase(), data.getHiveTable().toLowerCase(),
clusterName));
         return name.toString();
     }
 
@@ -149,6 +155,10 @@ public class SqoopHook extends SqoopJobDataPublisher {
         return name.toString();
     }
 
+    static boolean isImportOperation(SqoopJobDataPublisher.Data data) {
+        return data.getOperation().toLowerCase().equals("import");
+    }
+
     @Override
     public void publish(SqoopJobDataPublisher.Data data) throws Exception {
         Configuration atlasProperties = ApplicationProperties.get();

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/5b627b5a/addons/sqoop-bridge/src/test/java/org/apache/atlas/sqoop/hook/SqoopHookIT.java
----------------------------------------------------------------------
diff --git a/addons/sqoop-bridge/src/test/java/org/apache/atlas/sqoop/hook/SqoopHookIT.java
b/addons/sqoop-bridge/src/test/java/org/apache/atlas/sqoop/hook/SqoopHookIT.java
index a81ee15..5256439 100644
--- a/addons/sqoop-bridge/src/test/java/org/apache/atlas/sqoop/hook/SqoopHookIT.java
+++ b/addons/sqoop-bridge/src/test/java/org/apache/atlas/sqoop/hook/SqoopHookIT.java
@@ -88,6 +88,21 @@ public class SqoopHookIT {
         assertHiveTableIsRegistered(DEFAULT_DB, "hiveTable");
     }
 
+    @Test
+    public void testSqoopExport() throws Exception {
+        SqoopJobDataPublisher.Data d = new SqoopJobDataPublisher.Data("export", "jdbc:mysql:///localhost/db",
+                "mysqluser", "mysql", "myTable", null, "default", "hiveTable", new Properties(),
+                System.currentTimeMillis() - 100, System.currentTimeMillis());
+        SqoopHook hook = new SqoopHook();
+        hook.publish(d);
+        Thread.sleep(1000);
+        String storeName  = SqoopHook.getSqoopDBStoreName(d);
+        assertDBStoreIsRegistered(storeName);
+        String name = SqoopHook.getSqoopProcessName(d, CLUSTER_NAME);
+        assertSqoopProcessIsRegistered(name);
+        assertHiveTableIsRegistered(DEFAULT_DB, "hiveTable");
+    }
+
     private String assertDBStoreIsRegistered(String storeName) throws Exception {
         LOG.debug("Searching for db store {}",  storeName);
         String query = String.format(

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/5b627b5a/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 56b3472..8cbdecc 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -22,6 +22,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file
(dosset
 ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via
shwethags)
 
 ALL CHANGES:
+ATLAS-837 Enhance Sqoop addon to handle export operation (venkatnrangan via shwethags)
 ATLAS-869 Make LDAP/AD properties to be configurable. (nixonrodrigues via yhemanth)
 ATLAS-730 Change titan hbase table name (svimal2106 via sumasai )
 ATLAS-871 Make audit repository implementation configurable (jnhagelb via shwethags)


Mime
View raw message