Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 4F066200B43 for ; Tue, 19 Jul 2016 20:43:58 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 4D8F1160A5C; Tue, 19 Jul 2016 18:43:58 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id D8FBC160A76 for ; Tue, 19 Jul 2016 20:43:55 +0200 (CEST) Received: (qmail 44080 invoked by uid 500); 19 Jul 2016 18:43:55 -0000 Mailing-List: contact commits-help@falcon.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@falcon.apache.org Delivered-To: mailing list commits@falcon.apache.org Received: (qmail 44004 invoked by uid 99); 19 Jul 2016 18:43:55 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 19 Jul 2016 18:43:55 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id DC33BE058E; Tue, 19 Jul 2016 18:43:54 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: balu@apache.org To: commits@falcon.apache.org Date: Tue, 19 Jul 2016 18:43:55 -0000 Message-Id: In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [2/2] falcon git commit: FALCON-2066 Extend support for Import and Export of various database types archived-at: Tue, 19 Jul 2016 18:43:58 -0000 FALCON-2066 Extend support for Import and Export of various database types This is a replacement for pull request - https://github.com/apache/falcon/pull/224. + Checkstyle violations addressed + Removed the changes that went into pull-218 + Created https://issues.apache.org/jira/browse/FALCON-2083 to track the direct mode & verbose + Removed tab from the xml files Author: Venkatesan Ramachandran Reviewers: "Peeyush B , Balu Vellanki " Closes #229 from vramachan/FALCON-2066.DatabaseTypes Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/292dfed3 Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/292dfed3 Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/292dfed3 Branch: refs/heads/master Commit: 292dfed341b462488ab58584dc305f519f1beb4d Parents: 9a4c18c Author: Venkatesan Ramachandran Authored: Tue Jul 19 11:43:50 2016 -0700 Committer: bvellanki Committed: Tue Jul 19 11:43:50 2016 -0700 ---------------------------------------------------------------------- client/src/main/resources/datasource-0.1.xsd | 5 ++ docs/src/site/twiki/EntitySpecification.twiki | 8 ++ examples/entity/importexport/README.txt | 56 ++++++++++++ examples/entity/importexport/db2-datasource.xml | 56 ++++++++++++ .../importexport/db2-feed-import-filesystem.xml | 71 ++++++++++++++++ .../importexport/feed-export-filesystem.xml | 63 ++++++++++++++ .../entity/importexport/feed-export-hcat.xml | 65 ++++++++++++++ .../feed-export-repl-filesystem.xml | 72 ++++++++++++++++ .../feed-import-export-filesystem.xml | 80 ++++++++++++++++++ .../feed-import-export-repl-filesystem.xml | 89 ++++++++++++++++++++ .../entity/importexport/feed-import-hcat.xml | 69 +++++++++++++++ .../feed-import-repl-filesystem.xml | 81 ++++++++++++++++++ .../entity/importexport/generic-datasource.xml | 49 +++++++++++ .../generic-feed-import-filesystem.xml | 72 ++++++++++++++++ .../entity/importexport/mysql-datasource.xml | 58 +++++++++++++ .../mysql-feed-export-filesystem.xml | 62 ++++++++++++++ .../mysql-feed-import-filesystem.xml | 72 ++++++++++++++++ .../entity/importexport/netezza-datasource.xml | 57 +++++++++++++ .../netezza-feed-import-filesystem.xml | 72 ++++++++++++++++ .../entity/importexport/oracle-datasource.xml | 50 +++++++++++ .../oracle-feed-import-filesystem.xml | 72 ++++++++++++++++ .../entity/importexport/pgsql-datasource.xml | 55 ++++++++++++ .../pgsql-feed-import-filesystem.xml | 70 +++++++++++++++ .../entity/importexport/primary_cluster.xml | 33 ++++++++ examples/entity/importexport/td-datasource.xml | 56 ++++++++++++ .../importexport/td-feed-import-filesystem.xml | 70 +++++++++++++++ .../oozie/DatabaseExportWorkflowBuilder.java | 28 ++---- .../oozie/DatabaseImportWorkflowBuilder.java | 37 ++------ .../apache/falcon/oozie/ImportExportCommon.java | 22 ++++- .../OozieOrchestrationWorkflowBuilder.java | 21 +---- 30 files changed, 1603 insertions(+), 68 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/client/src/main/resources/datasource-0.1.xsd ---------------------------------------------------------------------- diff --git a/client/src/main/resources/datasource-0.1.xsd b/client/src/main/resources/datasource-0.1.xsd index ef78239..d78d967 100644 --- a/client/src/main/resources/datasource-0.1.xsd +++ b/client/src/main/resources/datasource-0.1.xsd @@ -232,6 +232,11 @@ + + + + + http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/docs/src/site/twiki/EntitySpecification.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/EntitySpecification.twiki b/docs/src/site/twiki/EntitySpecification.twiki index faad305..2615e5d 100644 --- a/docs/src/site/twiki/EntitySpecification.twiki +++ b/docs/src/site/twiki/EntitySpecification.twiki @@ -117,6 +117,14 @@ Following are the tags defined in a datasource.xml: name="test-hsql-db" xmlns="uri:falcon:datasource:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> +---+++ Datasource Types + +Falcon currently supports relational databases as data sources (both source and target). The following relational +databases are supported + * MySQL, HSQL, Postgres, Oracle, Teradata, Netezza, DB2 + * Generic - generic jdbc data source. This will require specifying a driver classname and jar file in the + datasource entity specification. Please see samples in the examples dir. + The colo specifies the colo to which the datasource belongs to and name is the name of the datasource which has to be unique. http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/README.txt ---------------------------------------------------------------------- diff --git a/examples/entity/importexport/README.txt b/examples/entity/importexport/README.txt new file mode 100755 index 0000000..8b20aa5 --- /dev/null +++ b/examples/entity/importexport/README.txt @@ -0,0 +1,56 @@ +####### Using MySQL as Datasource ####### +yum install mysql mysql-server mysql-libs +/usr/share/java/mysql-connector-java.jar +cp /home/ambari-qa/falcon_ingestion/backup/mysql-connector-java-5.1.31.jar /usr/share/java/ +unlink /usr/share/java/mysql-connector-java.jar +ln -s /home/ambari-qa/falcon_ingestion/backup/mysql-connector-java-5.1.31.jar /usr/share/java/mysql-connector-java.jar +service mysqld start + +####### oozie sharelib change for sqoop ###################### +# Copy /usr/share/java/mysql-connector-java.jar (version 5.1.31) into oozie share lib of sqoop dir +hadoop dfs -put /usr/share/java/mysql-connector-java-5.1.31.jar /user/oozie/share/lib/lib_XXXXXXXXXXX/sqoop + +####### Setup MySQL user and create source table ############## +# Setup - Create MySQL source table - db_raw_data +############################################################### +use test; +create user sqoop_user@localhost identified by 'sqoop'; +grant all privileges on *.* to 'sqoop_user'@'localhost' with grant option; +create user 'sqoop_user'@'%' identified by 'sqoop'; +grant all privileges on *.* to 'sqoop_user'@'%' with grant option; +flush privileges; + +## Import: following is the data source tables from which data will be pulled into HADOOP + +create table db_raw_data(id int not null, name varchar(20), value int, modified_ts timestamp); +insert into db_raw_data values (1, 'Apple', 700, now() ); +insert into db_raw_data values (2, 'Blackberry', 1, now() ); +insert into db_raw_data values (3, 'Cisco', 100, now() ); +insert into db_raw_data values (4, 'Delta', 10, now() ); +insert into db_raw_data values (5, 'Eagle', 55, now() ); +insert into db_raw_data values (6, 'Falcon', 99, now() ); + +## submit the following falcon entities and schedule + +falcon entity -type cluster -file primary_cluster.xml -submit +falcon entity -type datasource -file mysql-datasource.xml -submit +falcon entity -type feed -file mysql-feed-import-filesystem.xml -submitAndSchedule + +## Export: create a target table in the MySQL database + +create table db_export_fs(id int not null, name varchar(20), value int, modified_ts timestamp); + +## submit the following falcon entities and schedule +falcon entity -type feed -file mysql-feed-export-filesystem.xml -submitAndSchedule + +### Note +# For other supported database types like Oracle, Postgres, Teradata, Netezza, DB2, Generic JDB +# please use the corresponding feed entity that starts with the database name in the xml entity +# definition. + +# Also, please make sure to copy the jdbc driver onto the Oozie Sqoop shared lib on HDFS and +# restart Oozie. The datasource entity definition will also refer to the JDBC drivers in order +# for falcon to validate the connection when submitting the datasource entity. + + + http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/db2-datasource.xml ---------------------------------------------------------------------- diff --git a/examples/entity/importexport/db2-datasource.xml b/examples/entity/importexport/db2-datasource.xml new file mode 100755 index 0000000..36a061b --- /dev/null +++ b/examples/entity/importexport/db2-datasource.xml @@ -0,0 +1,56 @@ + + + + + owner=foobar@ambari.apache.org, consumer=phoe@ambari.apache.org + + + + + + sqoop + sqoop + + + + + + + sqoop + sqoop + + + + + + sqoop + + sqoop.password.alias + hdfs://c6401.ambari.apache.org:8020/user/ambari-qa/sqoop_password.jceks + + + + + + + com.ibm.db2.jcc.DB2Driver + /user/oozie/share/lib/lib_20160628052542/sqoop/db2jcc4.jar + + + http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/db2-feed-import-filesystem.xml ---------------------------------------------------------------------- diff --git a/examples/entity/importexport/db2-feed-import-filesystem.xml b/examples/entity/importexport/db2-feed-import-filesystem.xml new file mode 100755 index 0000000..2adca37 --- /dev/null +++ b/examples/entity/importexport/db2-feed-import-filesystem.xml @@ -0,0 +1,71 @@ + + + + + + externalSystem=USWestEmailServers,classification=secure + DataImportPipeline + minutes(10) + + + + + + + + + snapshot + + + + id + name + + + + + + + + + + + + + + + + + + + + + + + days(1) + default + LOW + + + + + + http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/feed-export-filesystem.xml ---------------------------------------------------------------------- diff --git a/examples/entity/importexport/feed-export-filesystem.xml b/examples/entity/importexport/feed-export-filesystem.xml new file mode 100755 index 0000000..36d2c79 --- /dev/null +++ b/examples/entity/importexport/feed-export-filesystem.xml @@ -0,0 +1,63 @@ + + + + + + + externalSystem=USWestEmailServers,classification=secure + DataImportPipeline + minutes(10) + + + + + + + + + + + + + + + + + + + + + + + + + + + + days(1) + default + LOW + + + + + + http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/feed-export-hcat.xml ---------------------------------------------------------------------- diff --git a/examples/entity/importexport/feed-export-hcat.xml b/examples/entity/importexport/feed-export-hcat.xml new file mode 100755 index 0000000..f52bf20 --- /dev/null +++ b/examples/entity/importexport/feed-export-hcat.xml @@ -0,0 +1,65 @@ + + + + + + + externalSystem=USWestEmailServers,classification=secure + DataImportPipeline + minutes(10) + + + + + + + + + + + + + + + + + + + + + + + + + days(1) + default + LOW + + + + + + http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/feed-export-repl-filesystem.xml ---------------------------------------------------------------------- diff --git a/examples/entity/importexport/feed-export-repl-filesystem.xml b/examples/entity/importexport/feed-export-repl-filesystem.xml new file mode 100755 index 0000000..1932b5b --- /dev/null +++ b/examples/entity/importexport/feed-export-repl-filesystem.xml @@ -0,0 +1,72 @@ + + + + + + + externalSystem=USWestEmailServers,classification=secure + DataImportPipeline + minutes(10) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + days(1) + default + LOW + + + + + + http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/feed-import-export-filesystem.xml ---------------------------------------------------------------------- diff --git a/examples/entity/importexport/feed-import-export-filesystem.xml b/examples/entity/importexport/feed-import-export-filesystem.xml new file mode 100755 index 0000000..4e9e215 --- /dev/null +++ b/examples/entity/importexport/feed-import-export-filesystem.xml @@ -0,0 +1,80 @@ + + + + + + + externalSystem=USWestEmailServers,classification=secure + DataImportPipeline + minutes(10) + + + + + + + + + snapshot + + + + id + name + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + days(1) + default + LOW + + + + + + http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/feed-import-export-repl-filesystem.xml ---------------------------------------------------------------------- diff --git a/examples/entity/importexport/feed-import-export-repl-filesystem.xml b/examples/entity/importexport/feed-import-export-repl-filesystem.xml new file mode 100755 index 0000000..4e9a9be --- /dev/null +++ b/examples/entity/importexport/feed-import-export-repl-filesystem.xml @@ -0,0 +1,89 @@ + + + + + + + externalSystem=USWestEmailServers,classification=secure + DataImportPipeline + minutes(10) + + + + + + + + + snapshot + + + + id + name + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + days(1) + default + LOW + + + + + + http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/feed-import-hcat.xml ---------------------------------------------------------------------- diff --git a/examples/entity/importexport/feed-import-hcat.xml b/examples/entity/importexport/feed-import-hcat.xml new file mode 100755 index 0000000..73d2bf4 --- /dev/null +++ b/examples/entity/importexport/feed-import-hcat.xml @@ -0,0 +1,69 @@ + + + + + + + externalSystem=USWestEmailServers,classification=secure + DataImportPipeline + minutes(10) + + + + + + + + + snapshot + + + + id + name + + + + + + + + + + + + +
+ + + + + + + days(1) + default + LOW + + + + + + http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/feed-import-repl-filesystem.xml ---------------------------------------------------------------------- diff --git a/examples/entity/importexport/feed-import-repl-filesystem.xml b/examples/entity/importexport/feed-import-repl-filesystem.xml new file mode 100755 index 0000000..2bc110a --- /dev/null +++ b/examples/entity/importexport/feed-import-repl-filesystem.xml @@ -0,0 +1,81 @@ + + + + + + + externalSystem=USWestEmailServers,classification=secure + DataImportPipeline + minutes(10) + + + + + + + + + snapshot + + + + id + name + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + days(1) + default + LOW + + + + + + http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/generic-datasource.xml ---------------------------------------------------------------------- diff --git a/examples/entity/importexport/generic-datasource.xml b/examples/entity/importexport/generic-datasource.xml new file mode 100755 index 0000000..74f15ca --- /dev/null +++ b/examples/entity/importexport/generic-datasource.xml @@ -0,0 +1,49 @@ + + + + + + owner=foobar@ambari.apache.org, consumer=phoe@ambari.apache.org + + + + + + sqoop_user + sqoop + + + + + + + sqoop2_user + sqoop + + + + + + + com.mysql.jdbc.Driver + /user/oozie/share/lib/lib_20160628052542/sqoop/mysql-connector-java-5.1.31.jar + + + + http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/generic-feed-import-filesystem.xml ---------------------------------------------------------------------- diff --git a/examples/entity/importexport/generic-feed-import-filesystem.xml b/examples/entity/importexport/generic-feed-import-filesystem.xml new file mode 100755 index 0000000..01ccdab --- /dev/null +++ b/examples/entity/importexport/generic-feed-import-filesystem.xml @@ -0,0 +1,72 @@ + + + + + + + externalSystem=USWestEmailServers,classification=secure + DataImportPipeline + minutes(10) + + + + + + + + + snapshot + + + + id + name + + + + + + + + + + + + + + + + + + + + + + + days(1) + default + LOW + + + + + + http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/mysql-datasource.xml ---------------------------------------------------------------------- diff --git a/examples/entity/importexport/mysql-datasource.xml b/examples/entity/importexport/mysql-datasource.xml new file mode 100755 index 0000000..293b0bb --- /dev/null +++ b/examples/entity/importexport/mysql-datasource.xml @@ -0,0 +1,58 @@ + + + + + + owner=foobar@ambari.apache.org, consumer=phoe@ambari.apache.org + + + + + + sqoop_user + sqoop + + + + + + + sqoop2_user + sqoop + + + + + + sqoop2_user + + sqoop.password.alias + hdfs://c6407.ambari.apache.org:8020/user/ambari-qa/sqoop_password.jceks + + + + + + + com.mysql.jdbc.Driver + /user/oozie/share/lib/lib_20160628052542/sqoop/mysql-connector-java-5.1.31.jar + + + + http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/mysql-feed-export-filesystem.xml ---------------------------------------------------------------------- diff --git a/examples/entity/importexport/mysql-feed-export-filesystem.xml b/examples/entity/importexport/mysql-feed-export-filesystem.xml new file mode 100755 index 0000000..2e07aa2 --- /dev/null +++ b/examples/entity/importexport/mysql-feed-export-filesystem.xml @@ -0,0 +1,62 @@ + +# + + + + externalSystem=USWestEmailServers,classification=secure + DataImportPipeline + minutes(10) + + + + + + + + + + + + + + + + + + + + + + + + + + + + days(1) + default + LOW + + + + + + http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/mysql-feed-import-filesystem.xml ---------------------------------------------------------------------- diff --git a/examples/entity/importexport/mysql-feed-import-filesystem.xml b/examples/entity/importexport/mysql-feed-import-filesystem.xml new file mode 100755 index 0000000..b02b86f --- /dev/null +++ b/examples/entity/importexport/mysql-feed-import-filesystem.xml @@ -0,0 +1,72 @@ + + + + + + + externalSystem=USWestEmailServers,classification=secure + DataImportPipeline + minutes(10) + + + + + + + + + snapshot + + + + id + name + + + + + + + + + + + + + + + + + + + + + + + days(1) + default + LOW + + + + + + http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/netezza-datasource.xml ---------------------------------------------------------------------- diff --git a/examples/entity/importexport/netezza-datasource.xml b/examples/entity/importexport/netezza-datasource.xml new file mode 100755 index 0000000..1e4dfe6 --- /dev/null +++ b/examples/entity/importexport/netezza-datasource.xml @@ -0,0 +1,57 @@ + + + + + + owner=foobar@ambari.apache.org, consumer=phoe@ambari.apache.org + + + + + + admin + password + + + + + + + admin + password + + + + + + admin + + sqoop.password.alias + hdfs://c6401.ambari.apache.org:8020/user/ambari-qa/sqoop_password.jceks + + + + + + + org.netezza.Driver + /user/oozie/share/lib/lib_20160628052542/sqoop/nzjdbc.jar + + + http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/netezza-feed-import-filesystem.xml ---------------------------------------------------------------------- diff --git a/examples/entity/importexport/netezza-feed-import-filesystem.xml b/examples/entity/importexport/netezza-feed-import-filesystem.xml new file mode 100755 index 0000000..bbbcbad --- /dev/null +++ b/examples/entity/importexport/netezza-feed-import-filesystem.xml @@ -0,0 +1,72 @@ + + + + + + + externalSystem=USWestEmailServers,classification=secure + DataImportPipeline + minutes(10) + + + + + + + + + snapshot + + + + id + name + + + + + + + + + + + + + + + + + + + + + + + days(1) + default + LOW + + + + + + http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/oracle-datasource.xml ---------------------------------------------------------------------- diff --git a/examples/entity/importexport/oracle-datasource.xml b/examples/entity/importexport/oracle-datasource.xml new file mode 100755 index 0000000..535640c --- /dev/null +++ b/examples/entity/importexport/oracle-datasource.xml @@ -0,0 +1,50 @@ + + + + + + owner=foobar@ambari.apache.org, consumer=phoe@ambari.apache.org + + + + + + SQOOP + sqoop + + + + + + + SQOOP + sqoop + + + + + + + oracle.jdbc.OracleDriver + /user/oozie/share/lib/lib_20160628052542/sqoop/ojdbc6.jar + /user/oozie/share/lib/lib_20160628052542/sqoop/orai18n.jar + + + + http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/oracle-feed-import-filesystem.xml ---------------------------------------------------------------------- diff --git a/examples/entity/importexport/oracle-feed-import-filesystem.xml b/examples/entity/importexport/oracle-feed-import-filesystem.xml new file mode 100755 index 0000000..5bba3de --- /dev/null +++ b/examples/entity/importexport/oracle-feed-import-filesystem.xml @@ -0,0 +1,72 @@ + + + + + + + externalSystem=USWestEmailServers,classification=secure + DataImportPipeline + minutes(10) + + + + + + + + + snapshot + + + + ID + NAME + + + + + + + + + + + + + + + + + + + + + + + days(1) + default + LOW + + + + + + http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/pgsql-datasource.xml ---------------------------------------------------------------------- diff --git a/examples/entity/importexport/pgsql-datasource.xml b/examples/entity/importexport/pgsql-datasource.xml new file mode 100755 index 0000000..b161bd3 --- /dev/null +++ b/examples/entity/importexport/pgsql-datasource.xml @@ -0,0 +1,55 @@ + + + + + + owner=foobar@ambari.apache.org, consumer=phoe@ambari.apache.org + + + + + ambari-qa + sqoop + + + + + + + ambari-qa + sqoop + + + + + + ambari-qa + + sqoop.password.alias + hdfs://c6401.ambari.apache.org:8020/user/ambari-qa/sqoop_password.jceks + + + + + + org.postgresql.Driver + /user/oozie/share/lib/lib_20160628052542/sqoop/postgresql-9.4.1208.jar + + + http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/pgsql-feed-import-filesystem.xml ---------------------------------------------------------------------- diff --git a/examples/entity/importexport/pgsql-feed-import-filesystem.xml b/examples/entity/importexport/pgsql-feed-import-filesystem.xml new file mode 100755 index 0000000..fa88cd1 --- /dev/null +++ b/examples/entity/importexport/pgsql-feed-import-filesystem.xml @@ -0,0 +1,70 @@ + + + + + externalSystem=USWestEmailServers,classification=secure + DataImportPipeline + minutes(10) + + + + + + + + + snapshot + + + + id + name + + + + + + + + + + + + + + + + + + + + + + + days(1) + default + LOW + + + + + + http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/primary_cluster.xml ---------------------------------------------------------------------- diff --git a/examples/entity/importexport/primary_cluster.xml b/examples/entity/importexport/primary_cluster.xml new file mode 100755 index 0000000..4d43fd7 --- /dev/null +++ b/examples/entity/importexport/primary_cluster.xml @@ -0,0 +1,33 @@ + + + + + + + + + + + + + + + + + http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/td-datasource.xml ---------------------------------------------------------------------- diff --git a/examples/entity/importexport/td-datasource.xml b/examples/entity/importexport/td-datasource.xml new file mode 100755 index 0000000..a9eb4ed --- /dev/null +++ b/examples/entity/importexport/td-datasource.xml @@ -0,0 +1,56 @@ + + + + owner=foobar@ambari.apache.org, consumer=phoe@ambari.apache.org + + + + + + dbc + dbc + + + + + + + dbc + dbc + + + + + + dbc + + sqoop.password.alias + hdfs://c6401.ambari.apache.org:8020/user/ambari-qa/sqoop_password.jceks + + + + + + + com.teradata.jdbc.TeraDriver + /user/oozie/share/lib/lib_20160628052542/sqoop/terajdbc4.jar + /user/oozie/share/lib/lib_20160628052542/sqoop/tdgssconfig.jar + + + http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/td-feed-import-filesystem.xml ---------------------------------------------------------------------- diff --git a/examples/entity/importexport/td-feed-import-filesystem.xml b/examples/entity/importexport/td-feed-import-filesystem.xml new file mode 100755 index 0000000..6dd94ed --- /dev/null +++ b/examples/entity/importexport/td-feed-import-filesystem.xml @@ -0,0 +1,70 @@ + + + + + externalSystem=USWestEmailServers,classification=secure + DataImportPipeline + minutes(10) + + + + + + + + + snapshot + + + + id + name + + + + + + + + + + + + + + + + + + + + + + + days(1) + default + LOW + + + + + + http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/oozie/src/main/java/org/apache/falcon/oozie/DatabaseExportWorkflowBuilder.java ---------------------------------------------------------------------- diff --git a/oozie/src/main/java/org/apache/falcon/oozie/DatabaseExportWorkflowBuilder.java b/oozie/src/main/java/org/apache/falcon/oozie/DatabaseExportWorkflowBuilder.java index e7ca869..93f3d1f 100644 --- a/oozie/src/main/java/org/apache/falcon/oozie/DatabaseExportWorkflowBuilder.java +++ b/oozie/src/main/java/org/apache/falcon/oozie/DatabaseExportWorkflowBuilder.java @@ -94,12 +94,15 @@ public class DatabaseExportWorkflowBuilder extends ExportWorkflowBuilder { StringBuilder sqoopArgs = new StringBuilder(); StringBuilder sqoopOptions = new StringBuilder(); - buildConnectArg(sqoopArgs, cluster).append(ImportExportCommon.ARG_SEPARATOR); - buildTableArg(sqoopArgs, cluster).append(ImportExportCommon.ARG_SEPARATOR); - Datasource datasource = DatasourceHelper.getDatasource(FeedHelper.getExportDatasourceName( - FeedHelper.getCluster(entity, cluster.getName()))); - ImportExportCommon.buildUserPasswordArg(sqoopArgs, sqoopOptions, datasource) - .append(ImportExportCommon.ARG_SEPARATOR); + org.apache.falcon.entity.v0.feed.Cluster feedCluster = FeedHelper.getCluster(entity, cluster.getName()); + Datasource datasource = DatasourceHelper.getDatasource(FeedHelper.getExportDatasourceName(feedCluster)); + ImportExportCommon.buildDriverArgs(sqoopArgs, datasource).append(ImportExportCommon.ARG_SEPARATOR); + ImportExportCommon.buildConnectArg(sqoopArgs, DatasourceHelper.getWriteEndpoint(datasource)) + .append(ImportExportCommon.ARG_SEPARATOR); + ImportExportCommon.buildTableArg(sqoopArgs, FeedHelper.getExportDataSourceTableName(feedCluster)) + .append(ImportExportCommon.ARG_SEPARATOR); + ImportExportCommon.buildUserPasswordArg(sqoopArgs, sqoopOptions, + DatasourceHelper.getWritePasswordInfo(datasource)).append(ImportExportCommon.ARG_SEPARATOR); buildNumMappers(sqoopArgs, extraArgs).append(ImportExportCommon.ARG_SEPARATOR); buildArguments(sqoopArgs, extraArgs, feed, cluster).append(ImportExportCommon.ARG_SEPARATOR); buildLoadType(sqoopArgs, cluster).append(ImportExportCommon.ARG_SEPARATOR); @@ -111,19 +114,6 @@ public class DatabaseExportWorkflowBuilder extends ExportWorkflowBuilder { .append(sqoopArgs).toString(); } - private StringBuilder buildConnectArg(StringBuilder builder, Cluster cluster) throws FalconException { - org.apache.falcon.entity.v0.feed.Cluster feedCluster = FeedHelper.getCluster(entity, cluster.getName()); - return builder.append("--connect").append(ImportExportCommon.ARG_SEPARATOR) - .append(DatasourceHelper.getReadOnlyEndpoint( - DatasourceHelper.getDatasource(FeedHelper.getExportDatasourceName(feedCluster)))); - } - - private StringBuilder buildTableArg(StringBuilder builder, Cluster cluster) throws FalconException { - org.apache.falcon.entity.v0.feed.Cluster feedCluster = FeedHelper.getCluster(entity, cluster.getName()); - return builder.append("--table").append(ImportExportCommon.ARG_SEPARATOR) - .append(FeedHelper.getExportDataSourceTableName(feedCluster)); - } - private StringBuilder buildLoadType(StringBuilder builder, Cluster cluster) throws FalconException { org.apache.falcon.entity.v0.feed.Cluster feedCluster = FeedHelper.getCluster(entity, cluster.getName()); http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/oozie/src/main/java/org/apache/falcon/oozie/DatabaseImportWorkflowBuilder.java ---------------------------------------------------------------------- diff --git a/oozie/src/main/java/org/apache/falcon/oozie/DatabaseImportWorkflowBuilder.java b/oozie/src/main/java/org/apache/falcon/oozie/DatabaseImportWorkflowBuilder.java index 1efa368..77d5462 100644 --- a/oozie/src/main/java/org/apache/falcon/oozie/DatabaseImportWorkflowBuilder.java +++ b/oozie/src/main/java/org/apache/falcon/oozie/DatabaseImportWorkflowBuilder.java @@ -95,13 +95,16 @@ public class DatabaseImportWorkflowBuilder extends ImportWorkflowBuilder { Map extraArgs = getArguments(cluster); StringBuilder sqoopArgs = new StringBuilder(); StringBuilder sqoopOptions = new StringBuilder(); - buildDriverArgs(sqoopArgs, cluster).append(ImportExportCommon.ARG_SEPARATOR); - buildConnectArg(sqoopArgs, cluster).append(ImportExportCommon.ARG_SEPARATOR); - buildTableArg(sqoopArgs, cluster).append(ImportExportCommon.ARG_SEPARATOR); - Datasource datasource = DatasourceHelper.getDatasource(FeedHelper.getImportDatasourceName( - FeedHelper.getCluster(entity, cluster.getName()))); - ImportExportCommon.buildUserPasswordArg(sqoopArgs, sqoopOptions, datasource) + + org.apache.falcon.entity.v0.feed.Cluster feedCluster = FeedHelper.getCluster(entity, cluster.getName()); + Datasource datasource = DatasourceHelper.getDatasource(FeedHelper.getImportDatasourceName(feedCluster)); + ImportExportCommon.buildDriverArgs(sqoopArgs, datasource).append(ImportExportCommon.ARG_SEPARATOR); + ImportExportCommon.buildConnectArg(sqoopArgs, DatasourceHelper.getReadOnlyEndpoint(datasource)) .append(ImportExportCommon.ARG_SEPARATOR); + ImportExportCommon.buildTableArg(sqoopArgs, FeedHelper.getImportDataSourceTableName(feedCluster)) + .append(ImportExportCommon.ARG_SEPARATOR); + ImportExportCommon.buildUserPasswordArg(sqoopArgs, sqoopOptions, + DatasourceHelper.getReadPasswordInfo(datasource)).append(ImportExportCommon.ARG_SEPARATOR); buildNumMappers(sqoopArgs, extraArgs).append(ImportExportCommon.ARG_SEPARATOR); buildArguments(sqoopArgs, extraArgs).append(ImportExportCommon.ARG_SEPARATOR); buildTargetArg(sqoopArgs, feed, cluster).append(ImportExportCommon.ARG_SEPARATOR); @@ -112,28 +115,6 @@ public class DatabaseImportWorkflowBuilder extends ImportWorkflowBuilder { .append(sqoopArgs).toString(); } - private StringBuilder buildDriverArgs(StringBuilder builder, Cluster cluster) throws FalconException { - org.apache.falcon.entity.v0.feed.Cluster feedCluster = FeedHelper.getCluster(entity, cluster.getName()); - Datasource db = DatasourceHelper.getDatasource(FeedHelper.getImportDatasourceName(feedCluster)); - if ((db.getDriver() != null) && (db.getDriver().getClazz() != null)) { - builder.append("--driver").append(ImportExportCommon.ARG_SEPARATOR).append(db.getDriver().getClazz()); - } - return builder; - } - - private StringBuilder buildConnectArg(StringBuilder builder, Cluster cluster) throws FalconException { - org.apache.falcon.entity.v0.feed.Cluster feedCluster = FeedHelper.getCluster(entity, cluster.getName()); - return builder.append("--connect").append(ImportExportCommon.ARG_SEPARATOR) - .append(DatasourceHelper.getReadOnlyEndpoint( - DatasourceHelper.getDatasource(FeedHelper.getImportDatasourceName(feedCluster)))); - } - - private StringBuilder buildTableArg(StringBuilder builder, Cluster cluster) throws FalconException { - org.apache.falcon.entity.v0.feed.Cluster feedCluster = FeedHelper.getCluster(entity, cluster.getName()); - return builder.append("--table").append(ImportExportCommon.ARG_SEPARATOR) - .append(FeedHelper.getImportDataSourceTableName(feedCluster)); - } - private StringBuilder buildTargetArg(StringBuilder builder, Feed feed, Cluster cluster) throws FalconException { Storage.TYPE feedStorageType = FeedHelper.getStorageType(feed, cluster);