falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From b...@apache.org
Subject [2/2] falcon git commit: FALCON-2066 Extend support for Import and Export of various database types
Date Tue, 19 Jul 2016 18:43:55 GMT
FALCON-2066 Extend support for Import and Export of various database types

This is a replacement for pull request - https://github.com/apache/falcon/pull/224.

+ Checkstyle violations addressed
+ Removed the changes that went into pull-218
+ Created https://issues.apache.org/jira/browse/FALCON-2083 to track the direct mode & verbose
+ Removed tab from the xml files

Author: Venkatesan Ramachandran <vramachandran@hortonworks.com>

Reviewers: "Peeyush B <pbishnoi@hortonworks.com>, Balu Vellanki <balu@apache.org>"

Closes #229 from vramachan/FALCON-2066.DatabaseTypes


Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/292dfed3
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/292dfed3
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/292dfed3

Branch: refs/heads/master
Commit: 292dfed341b462488ab58584dc305f519f1beb4d
Parents: 9a4c18c
Author: Venkatesan Ramachandran <vramachandran@hortonworks.com>
Authored: Tue Jul 19 11:43:50 2016 -0700
Committer: bvellanki <bvellanki@hortonworks.com>
Committed: Tue Jul 19 11:43:50 2016 -0700

----------------------------------------------------------------------
 client/src/main/resources/datasource-0.1.xsd    |  5 ++
 docs/src/site/twiki/EntitySpecification.twiki   |  8 ++
 examples/entity/importexport/README.txt         | 56 ++++++++++++
 examples/entity/importexport/db2-datasource.xml | 56 ++++++++++++
 .../importexport/db2-feed-import-filesystem.xml | 71 ++++++++++++++++
 .../importexport/feed-export-filesystem.xml     | 63 ++++++++++++++
 .../entity/importexport/feed-export-hcat.xml    | 65 ++++++++++++++
 .../feed-export-repl-filesystem.xml             | 72 ++++++++++++++++
 .../feed-import-export-filesystem.xml           | 80 ++++++++++++++++++
 .../feed-import-export-repl-filesystem.xml      | 89 ++++++++++++++++++++
 .../entity/importexport/feed-import-hcat.xml    | 69 +++++++++++++++
 .../feed-import-repl-filesystem.xml             | 81 ++++++++++++++++++
 .../entity/importexport/generic-datasource.xml  | 49 +++++++++++
 .../generic-feed-import-filesystem.xml          | 72 ++++++++++++++++
 .../entity/importexport/mysql-datasource.xml    | 58 +++++++++++++
 .../mysql-feed-export-filesystem.xml            | 62 ++++++++++++++
 .../mysql-feed-import-filesystem.xml            | 72 ++++++++++++++++
 .../entity/importexport/netezza-datasource.xml  | 57 +++++++++++++
 .../netezza-feed-import-filesystem.xml          | 72 ++++++++++++++++
 .../entity/importexport/oracle-datasource.xml   | 50 +++++++++++
 .../oracle-feed-import-filesystem.xml           | 72 ++++++++++++++++
 .../entity/importexport/pgsql-datasource.xml    | 55 ++++++++++++
 .../pgsql-feed-import-filesystem.xml            | 70 +++++++++++++++
 .../entity/importexport/primary_cluster.xml     | 33 ++++++++
 examples/entity/importexport/td-datasource.xml  | 56 ++++++++++++
 .../importexport/td-feed-import-filesystem.xml  | 70 +++++++++++++++
 .../oozie/DatabaseExportWorkflowBuilder.java    | 28 ++----
 .../oozie/DatabaseImportWorkflowBuilder.java    | 37 ++------
 .../apache/falcon/oozie/ImportExportCommon.java | 22 ++++-
 .../OozieOrchestrationWorkflowBuilder.java      | 21 +----
 30 files changed, 1603 insertions(+), 68 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/client/src/main/resources/datasource-0.1.xsd
----------------------------------------------------------------------
diff --git a/client/src/main/resources/datasource-0.1.xsd b/client/src/main/resources/datasource-0.1.xsd
index ef78239..d78d967 100644
--- a/client/src/main/resources/datasource-0.1.xsd
+++ b/client/src/main/resources/datasource-0.1.xsd
@@ -232,6 +232,11 @@
             <xs:enumeration value="mysql"/>
             <xs:enumeration value="oracle"/>
             <xs:enumeration value="hsql"/>
+            <xs:enumeration value="postgres"/>
+            <xs:enumeration value="db2"/>
+            <xs:enumeration value="netezza"/>
+            <xs:enumeration value="teradata"/>
+            <xs:enumeration value="generic"/>
         </xs:restriction>
     </xs:simpleType>
 

http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/docs/src/site/twiki/EntitySpecification.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/EntitySpecification.twiki b/docs/src/site/twiki/EntitySpecification.twiki
index faad305..2615e5d 100644
--- a/docs/src/site/twiki/EntitySpecification.twiki
+++ b/docs/src/site/twiki/EntitySpecification.twiki
@@ -117,6 +117,14 @@ Following are the tags defined in a datasource.xml:
  name="test-hsql-db" xmlns="uri:falcon:datasource:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
 </verbatim>
 
+---+++ Datasource Types
+
+Falcon currently supports relational databases as data sources (both source and target). The following relational
+databases are supported
+   * MySQL, HSQL, Postgres, Oracle, Teradata, Netezza, DB2
+   * Generic - generic jdbc data source. This will require specifying a driver classname and jar file in the
+       datasource entity specification. Please see samples in the examples dir.
+
 The colo specifies the colo to which the datasource belongs to and name is the name of the datasource which has to
 be unique.
 

http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/README.txt
----------------------------------------------------------------------
diff --git a/examples/entity/importexport/README.txt b/examples/entity/importexport/README.txt
new file mode 100755
index 0000000..8b20aa5
--- /dev/null
+++ b/examples/entity/importexport/README.txt
@@ -0,0 +1,56 @@
+####### Using MySQL as Datasource #######
+yum install mysql mysql-server mysql-libs
+/usr/share/java/mysql-connector-java.jar
+cp /home/ambari-qa/falcon_ingestion/backup/mysql-connector-java-5.1.31.jar /usr/share/java/
+unlink /usr/share/java/mysql-connector-java.jar
+ln -s /home/ambari-qa/falcon_ingestion/backup/mysql-connector-java-5.1.31.jar /usr/share/java/mysql-connector-java.jar
+service mysqld start
+
+####### oozie sharelib change for sqoop ######################
+# Copy /usr/share/java/mysql-connector-java.jar (version 5.1.31) into oozie share lib of sqoop dir
+hadoop dfs -put  /usr/share/java/mysql-connector-java-5.1.31.jar /user/oozie/share/lib/lib_XXXXXXXXXXX/sqoop
+
+####### Setup MySQL user and create source table ##############
+# Setup - Create MySQL source table - db_raw_data
+###############################################################
+use test;
+create user sqoop_user@localhost identified by 'sqoop';
+grant all privileges on *.* to 'sqoop_user'@'localhost' with grant option;
+create user 'sqoop_user'@'%' identified by 'sqoop';
+grant all privileges on *.* to 'sqoop_user'@'%' with grant option;
+flush privileges;
+
+## Import: following is the data source tables from which data will be pulled into HADOOP
+
+create table db_raw_data(id int not null, name varchar(20), value int, modified_ts timestamp);
+insert into db_raw_data values (1,  'Apple', 700, now() );
+insert into db_raw_data values (2,  'Blackberry', 1, now() );
+insert into db_raw_data values (3,  'Cisco', 100, now() );
+insert into db_raw_data values (4,  'Delta', 10, now() );
+insert into db_raw_data values (5,  'Eagle', 55, now() );
+insert into db_raw_data values (6,  'Falcon', 99, now() );
+
+## submit the following falcon entities and schedule
+
+falcon entity -type cluster -file primary_cluster.xml -submit
+falcon entity -type datasource -file mysql-datasource.xml -submit
+falcon entity -type feed -file mysql-feed-import-filesystem.xml -submitAndSchedule
+
+## Export: create a target table in the MySQL database 
+
+create table db_export_fs(id int not null, name varchar(20), value int, modified_ts timestamp);
+
+## submit the following falcon entities and schedule 
+falcon entity -type feed -file mysql-feed-export-filesystem.xml -submitAndSchedule
+
+### Note
+# For other supported database types like Oracle, Postgres, Teradata, Netezza, DB2, Generic JDB
+# please use the corresponding feed entity that starts with the database name in the xml entity
+# definition. 
+
+# Also, please make sure to copy the jdbc driver onto the Oozie Sqoop shared lib on HDFS and 
+# restart Oozie. The datasource entity definition will also refer to the JDBC drivers in order
+# for falcon to validate the connection when submitting the datasource entity.
+
+
+

http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/db2-datasource.xml
----------------------------------------------------------------------
diff --git a/examples/entity/importexport/db2-datasource.xml b/examples/entity/importexport/db2-datasource.xml
new file mode 100755
index 0000000..36a061b
--- /dev/null
+++ b/examples/entity/importexport/db2-datasource.xml
@@ -0,0 +1,56 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+
+<datasource colo="west-coast" description="DB2 database on west coast" type="db2" name="db2-db" xmlns="uri:falcon:datasource:0.1">
+    <tags>owner=foobar@ambari.apache.org, consumer=phoe@ambari.apache.org</tags>
+    <interfaces>
+ 
+        <!-- ***** read interface ***** -->
+        <interface type="readonly" endpoint="jdbc:db2://172.25.16.65:50000/Sample">
+            <credential type="password-text">
+                <userName>sqoop</userName>
+                <passwordText>sqoop</passwordText>
+            </credential>
+        </interface>
+
+        <!-- ***** write interface ***** -->
+        <interface type="write"  endpoint="jdbc:db2://172.25.16.65:50000/Sample">
+            <credential type="password-text">
+                <userName>sqoop</userName>
+                <passwordText>sqoop</passwordText>
+            </credential>
+        </interface>
+
+        <!-- ***** default credential ***** -->
+        <credential type="password-alias">
+            <userName>sqoop</userName>
+            <passwordAlias>
+                <alias>sqoop.password.alias</alias>
+                <providerPath>hdfs://c6401.ambari.apache.org:8020/user/ambari-qa/sqoop_password.jceks</providerPath>
+            </passwordAlias>
+        </credential>
+
+    </interfaces>
+
+    <driver>
+        <clazz>com.ibm.db2.jcc.DB2Driver</clazz>
+        <jar>/user/oozie/share/lib/lib_20160628052542/sqoop/db2jcc4.jar</jar>
+    </driver>
+</datasource>
+

http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/db2-feed-import-filesystem.xml
----------------------------------------------------------------------
diff --git a/examples/entity/importexport/db2-feed-import-filesystem.xml b/examples/entity/importexport/db2-feed-import-filesystem.xml
new file mode 100755
index 0000000..2adca37
--- /dev/null
+++ b/examples/entity/importexport/db2-feed-import-filesystem.xml
@@ -0,0 +1,71 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+
+<!--
+ A feed representing Hourly customer email data retained for 90 days
+ -->
+<feed description="Raw customer email feed" name="db2-feed-import-filesystem" xmlns="uri:falcon:feed:0.1">
+    <tags>externalSystem=USWestEmailServers,classification=secure</tags>
+    <groups>DataImportPipeline</groups>
+    <frequency>minutes(10)</frequency>
+    <late-arrival cut-off="hours(4)"/>
+    <clusters>
+        <cluster name="primaryCluster" type="source">
+            <validity start="2016-04-10T00:00Z" end="2017-03-31T00:00Z"/>
+            <retention limit="days(90)" action="delete"/>
+            <import> 
+                <source name="db2-db" tableName="db_raw_data">
+                    <extract type="full">
+                        <mergepolicy>snapshot</mergepolicy>
+                    </extract>    
+                    <fields>
+                           <includes>
+                             <field>id</field>
+                             <field>name</field>
+                           </includes>
+                    </fields>
+                </source>
+                <arguments>
+                    <argument name="--split-by" value="id"/>
+                    <argument name="--num-mappers" value="2"/>
+                </arguments>
+            </import>
+        </cluster>
+    </clusters>
+    
+    <locations>
+        <location type="data" path="/user/ambari-qa/falcon/demo/primary/db2-feed-import-filesystem/${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}"/>
+        <location type="stats" path="/none"/>
+        <location type="meta" path="/none"/>
+    </locations>
+    
+    <ACL owner="ambari-qa" group="users" permission="0755"/>
+    <schema location="/none" provider="none"/>
+
+   <lifecycle>
+        <retention-stage>
+            <frequency>days(1)</frequency>
+            <queue>default</queue>
+            <priority>LOW</priority>
+           <properties>
+                <property name="retention.policy.agebaseddelete.limit" value="hours(12)"></property>
+            </properties>
+        </retention-stage>
+    </lifecycle>
+</feed>

http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/feed-export-filesystem.xml
----------------------------------------------------------------------
diff --git a/examples/entity/importexport/feed-export-filesystem.xml b/examples/entity/importexport/feed-export-filesystem.xml
new file mode 100755
index 0000000..36d2c79
--- /dev/null
+++ b/examples/entity/importexport/feed-export-filesystem.xml
@@ -0,0 +1,63 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+
+<!--
+ A feed representing Hourly customer email data retained for 90 days
+ -->
+<feed description="Raw customer email feed" name="feed-export-filesystem" xmlns="uri:falcon:feed:0.1">
+    <tags>externalSystem=USWestEmailServers,classification=secure</tags>
+    <groups>DataImportPipeline</groups>
+    <frequency>minutes(10)</frequency>
+    <late-arrival cut-off="hours(4)"/>
+    <clusters>
+        <cluster name="primaryCluster" type="source">
+            <validity start="2016-01-01T00:00Z" end="2017-03-31T00:00Z"/>
+            <retention limit="days(90)" action="delete"/>
+            <export>
+                  <target name="mysql-db" tableName="simple_export_filesystem">
+                      <load type="allowinsert"/>
+                  </target>    
+                  <arguments>
+                       <argument name="--update-key" value="id"/>
+                  </arguments>
+              </export>
+        </cluster>
+    </clusters>
+    
+    <locations>
+        <location type="data" path="/user/ambari-qa/falcon/demo/primary/mysql-feed-import-filesystem/${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}"/>
+        <location type="stats" path="/none"/>
+        <location type="meta" path="/none"/>
+    </locations>
+    
+    <ACL owner="ambari-qa" group="users" permission="0755"/>
+    <schema location="/none" provider="none"/>
+
+   <lifecycle>
+        <retention-stage>
+            <frequency>days(1)</frequency>
+            <queue>default</queue>
+            <priority>LOW</priority>
+           <properties>
+                <property name="retention.policy.agebaseddelete.limit" value="hours(12)"></property>
+            </properties>
+        </retention-stage>
+    </lifecycle>
+</feed>

http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/feed-export-hcat.xml
----------------------------------------------------------------------
diff --git a/examples/entity/importexport/feed-export-hcat.xml b/examples/entity/importexport/feed-export-hcat.xml
new file mode 100755
index 0000000..f52bf20
--- /dev/null
+++ b/examples/entity/importexport/feed-export-hcat.xml
@@ -0,0 +1,65 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+
+<!--
+ A feed representing Hourly customer email data retained for 90 days
+ -->
+<feed description="Raw customer email feed" name="feed-export-hcat" xmlns="uri:falcon:feed:0.1">
+    <tags>externalSystem=USWestEmailServers,classification=secure</tags>
+    <groups>DataImportPipeline</groups>
+    <frequency>minutes(10)</frequency>
+    <late-arrival cut-off="hours(4)"/>
+    <clusters>
+        <cluster name="primaryCluster" type="source">
+            <validity start="2016-01-01T00:00Z" end="2016-03-31T00:00Z"/>
+            <retention limit="days(90)" action="delete"/>
+            <export>
+                  <target name="mysql-db" tableName="db_processed_data">
+                      <load type="allowinsert"/>
+                        <!--fields>
+                        <includes>
+                          <field>id</field>
+                          <field>name</field>
+                        </includes>
+                      </fields-->
+                  </target>    
+                  <arguments>
+                       <argument name="--update-key" value="id"/>
+                  </arguments>
+              </export>
+        </cluster>
+    </clusters>
+    
+    <table uri="catalog:default:hcat_raw_data#year=${YEAR};month=${MONTH};day=${DAY};hour=${HOUR};minute=${MINUTE}"/>
+
+    <ACL owner="ambari-qa" group="users" permission="0755"/>
+    <schema location="/none" provider="none"/>
+
+   <lifecycle>
+        <retention-stage>
+            <frequency>days(1)</frequency>
+            <queue>default</queue>
+            <priority>LOW</priority>
+           <properties>
+                <property name="retention.policy.agebaseddelete.limit" value="hours(12)"></property>
+            </properties>
+        </retention-stage>
+    </lifecycle>
+</feed>

http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/feed-export-repl-filesystem.xml
----------------------------------------------------------------------
diff --git a/examples/entity/importexport/feed-export-repl-filesystem.xml b/examples/entity/importexport/feed-export-repl-filesystem.xml
new file mode 100755
index 0000000..1932b5b
--- /dev/null
+++ b/examples/entity/importexport/feed-export-repl-filesystem.xml
@@ -0,0 +1,72 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+
+<!--
+ A feed representing Hourly customer email data retained for 90 days
+ -->
+<feed description="Raw customer email feed" name="feed-export-repl-filesystem" xmlns="uri:falcon:feed:0.1">
+    <tags>externalSystem=USWestEmailServers,classification=secure</tags>
+    <groups>DataImportPipeline</groups>
+    <frequency>minutes(10)</frequency>
+    <late-arrival cut-off="hours(4)"/>
+    <clusters>
+        <cluster name="primaryCluster" type="source">
+            <validity start="2016-01-01T00:00Z" end="2016-03-31T00:00Z"/>
+            <retention limit="days(90)" action="delete"/>
+            <export>
+                  <target name="mysql-db" tableName="simple_export_filesystem">
+                      <load type="allowinsert"/>
+                  </target>    
+                  <arguments>
+                       <argument name="--update-key" value="id"/>
+                  </arguments>
+              </export>
+        </cluster>
+        <cluster name="backupCluster" type="target">
+            <validity start="2016-01-01T00:00Z" end="2017-03-31T00:00Z"/>
+            <retention limit="days(90)" action="delete"/>
+            <locations>
+                <location type="data" path="/user/ambari-qa/falcon/demo/backup/feed-import-filesystem/${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}"/>
+                <location type="stats" path="/none"/>
+                <location type="meta" path="/none"/>
+            </locations>
+        </cluster>
+    </clusters>
+    
+    <locations>
+        <location type="data" path="/user/ambari-qa/falcon/demo/primary/feed-import-filesystem/${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}"/>
+        <location type="stats" path="/none"/>
+        <location type="meta" path="/none"/>
+    </locations>
+    
+    <ACL owner="ambari-qa" group="users" permission="0755"/>
+    <schema location="/none" provider="none"/>
+
+   <lifecycle>
+        <retention-stage>
+            <frequency>days(1)</frequency>
+            <queue>default</queue>
+            <priority>LOW</priority>
+           <properties>
+                <property name="retention.policy.agebaseddelete.limit" value="hours(12)"></property>
+            </properties>
+        </retention-stage>
+    </lifecycle>
+</feed>

http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/feed-import-export-filesystem.xml
----------------------------------------------------------------------
diff --git a/examples/entity/importexport/feed-import-export-filesystem.xml b/examples/entity/importexport/feed-import-export-filesystem.xml
new file mode 100755
index 0000000..4e9e215
--- /dev/null
+++ b/examples/entity/importexport/feed-import-export-filesystem.xml
@@ -0,0 +1,80 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+
+<!--
+ A feed representing Hourly customer email data retained for 90 days
+ -->
+<feed description="Raw customer email feed" name="feed-import-export-filesystem" xmlns="uri:falcon:feed:0.1">
+    <tags>externalSystem=USWestEmailServers,classification=secure</tags>
+    <groups>DataImportPipeline</groups>
+    <frequency>minutes(10)</frequency>
+    <late-arrival cut-off="hours(4)"/>
+    <clusters>
+        <cluster name="primaryCluster" type="source">
+            <validity start="2016-01-01T00:00Z" end="2016-03-31T00:00Z"/>
+            <retention limit="days(90)" action="delete"/>
+            <import> 
+                <source name="mysql-db" tableName="simple">
+                    <extract type="full">
+                        <mergepolicy>snapshot</mergepolicy>
+                    </extract>    
+                    <fields>
+                           <includes>
+                             <field>id</field>
+                             <field>name</field>
+                           </includes>
+                    </fields>
+                </source>
+                <arguments>
+                       <argument name="--split-by" value="id"/>
+                    <argument name="--num-mappers" value="2"/>
+                </arguments>
+            </import>
+            <export>
+                  <target name="mysql-db" tableName="simple_export">
+                      <load type="allowinsert"/>
+                  </target>    
+                  <arguments>
+                       <argument name="--update-key" value="id"/>
+                  </arguments>
+              </export>
+        </cluster>
+    </clusters>
+    
+    <locations>
+        <location type="data" path="/user/ambari-qa/falcon/demo/primary/feed-import-export/${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}"/>
+        <location type="stats" path="/none"/>
+        <location type="meta" path="/none"/>
+    </locations>
+    
+    <ACL owner="ambari-qa" group="users" permission="0755"/>
+    <schema location="/none" provider="none"/>
+
+   <lifecycle>
+        <retention-stage>
+            <frequency>days(1)</frequency>
+            <queue>default</queue>
+            <priority>LOW</priority>
+           <properties>
+                <property name="retention.policy.agebaseddelete.limit" value="hours(12)"></property>
+            </properties>
+        </retention-stage>
+    </lifecycle>
+</feed>

http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/feed-import-export-repl-filesystem.xml
----------------------------------------------------------------------
diff --git a/examples/entity/importexport/feed-import-export-repl-filesystem.xml b/examples/entity/importexport/feed-import-export-repl-filesystem.xml
new file mode 100755
index 0000000..4e9a9be
--- /dev/null
+++ b/examples/entity/importexport/feed-import-export-repl-filesystem.xml
@@ -0,0 +1,89 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+
+<!--
+ A feed representing Hourly customer email data retained for 90 days
+ -->
+<feed description="Raw customer email feed" name="feed-import-export-repl-filesystem" xmlns="uri:falcon:feed:0.1">
+    <tags>externalSystem=USWestEmailServers,classification=secure</tags>
+    <groups>DataImportPipeline</groups>
+    <frequency>minutes(10)</frequency>
+    <late-arrival cut-off="hours(4)"/>
+    <clusters>
+        <cluster name="primaryCluster" type="source">
+            <validity start="2016-01-01T00:00Z" end="2016-03-31T00:00Z"/>
+            <retention limit="days(90)" action="delete"/>
+            <import> 
+                <source name="mysql-db" tableName="simple">
+                    <extract type="full">
+                        <mergepolicy>snapshot</mergepolicy>
+                    </extract>    
+                    <fields>
+                           <includes>
+                             <field>id</field>
+                             <field>name</field>
+                           </includes>
+                    </fields>
+                </source>
+                <arguments>
+                       <argument name="--split-by" value="id"/>
+                    <argument name="--num-mappers" value="2"/>
+                </arguments>
+            </import>
+            <export>
+                  <target name="mysql-db" tableName="simple_import_export_filesystem">
+                      <load type="allowinsert"/>
+                  </target>    
+                  <arguments>
+                       <argument name="--update-key" value="id"/>
+                  </arguments>
+              </export>
+        </cluster>
+       <cluster name="backupCluster" type="target">
+            <validity start="2016-01-01T00:00Z" end="2017-03-31T00:00Z"/>
+            <retention limit="days(90)" action="delete"/>
+            <locations>
+                <location type="data" path="/user/ambari-qa/falcon/demo/backup/feed-import-export-filesystem/${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}"/>
+                <location type="stats" path="/none"/>
+                <location type="meta" path="/none"/>
+            </locations>
+        </cluster>
+    </clusters>
+    
+    <locations>
+        <location type="data" path="/user/ambari-qa/falcon/demo/primary/feed-import-export-filesystem/${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}"/>
+        <location type="stats" path="/none"/>
+        <location type="meta" path="/none"/>
+    </locations>
+    
+    <ACL owner="ambari-qa" group="users" permission="0755"/>
+    <schema location="/none" provider="none"/>
+
+   <lifecycle>
+        <retention-stage>
+            <frequency>days(1)</frequency>
+            <queue>default</queue>
+            <priority>LOW</priority>
+           <properties>
+                <property name="retention.policy.agebaseddelete.limit" value="hours(12)"></property>
+            </properties>
+        </retention-stage>
+    </lifecycle>
+</feed>

http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/feed-import-hcat.xml
----------------------------------------------------------------------
diff --git a/examples/entity/importexport/feed-import-hcat.xml b/examples/entity/importexport/feed-import-hcat.xml
new file mode 100755
index 0000000..73d2bf4
--- /dev/null
+++ b/examples/entity/importexport/feed-import-hcat.xml
@@ -0,0 +1,69 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+
+<!--
+ A feed representing Hourly customer email data retained for 90 days
+ -->
+<feed description="Raw customer email feed" name="feed-import-hcat" xmlns="uri:falcon:feed:0.1">
+    <tags>externalSystem=USWestEmailServers,classification=secure</tags>
+    <groups>DataImportPipeline</groups>
+    <frequency>minutes(10)</frequency>
+    <late-arrival cut-off="hours(4)"/>
+    <clusters>
+        <cluster name="primaryCluster" type="source">
+            <validity start="2016-01-01T00:00Z" end="2016-03-31T00:00Z"/>
+            <retention limit="days(90)" action="delete"/>
+            <import> 
+                <source name="mysql-db" tableName="mysql_raw_data">
+                    <extract type="full">
+                        <mergepolicy>snapshot</mergepolicy>
+                    </extract>    
+                    <fields>
+                           <includes>
+                             <field>id</field>
+                             <field>name</field>
+                           </includes>
+                    </fields>
+                </source>
+                <arguments>
+                       <argument name="--split-by" value="id"/>
+                    <argument name="--num-mappers" value="2"/>
+                </arguments>
+            </import>
+        </cluster>
+    </clusters>
+    
+    <!--<table uri="catalog:default:hcat_raw_data#year=${YEAR};month=${MONTH};day=${DAY};hour=${HOUR};minute=${MINUTE}"/>-->
+    <table uri="catalog:default:hcat_raw_data#year=BLAH"/>
+    
+    <ACL owner="ambari-qa" group="users" permission="0755"/>
+    <schema location="/none" provider="none"/>
+
+   <lifecycle>
+        <retention-stage>
+            <frequency>days(1)</frequency>
+            <queue>default</queue>
+            <priority>LOW</priority>
+           <properties>
+                <property name="retention.policy.agebaseddelete.limit" value="hours(12)"></property>
+            </properties>
+        </retention-stage>
+    </lifecycle>
+</feed>

http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/feed-import-repl-filesystem.xml
----------------------------------------------------------------------
diff --git a/examples/entity/importexport/feed-import-repl-filesystem.xml b/examples/entity/importexport/feed-import-repl-filesystem.xml
new file mode 100755
index 0000000..2bc110a
--- /dev/null
+++ b/examples/entity/importexport/feed-import-repl-filesystem.xml
@@ -0,0 +1,81 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+
+<!--
+ A feed representing Hourly customer email data retained for 90 days
+ -->
+<feed description="Raw customer email feed" name="feed-import-repl-filesystem" xmlns="uri:falcon:feed:0.1">
+    <tags>externalSystem=USWestEmailServers,classification=secure</tags>
+    <groups>DataImportPipeline</groups>
+    <frequency>minutes(10)</frequency>
+    <late-arrival cut-off="hours(4)"/>
+    <clusters>
+        <cluster name="primaryCluster" type="source">
+            <validity start="2016-01-01T00:00Z" end="2017-03-31T00:00Z"/>
+            <retention limit="days(90)" action="delete"/>
+            <import> 
+                <source name="mysql-db" tableName="mysql_raw_data">
+                    <extract type="full">
+                        <mergepolicy>snapshot</mergepolicy>
+                    </extract>    
+                    <fields>
+                           <includes>
+                             <field>id</field>
+                             <field>name</field>
+                           </includes>
+                    </fields>
+                </source>
+                <arguments>
+                       <argument name="--split-by" value="id"/>
+                    <argument name="--num-mappers" value="2"/>
+                </arguments>
+            </import>
+        </cluster>
+        <cluster name="backupCluster" type="target">
+            <validity start="2016-01-01T00:00Z" end="2017-03-31T00:00Z"/>
+            <retention limit="days(90)" action="delete"/>
+            <locations>
+                <location type="data" path="/user/ambari-qa/falcon/demo/backup/feed-import-filesystem/${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}"/>
+                <location type="stats" path="/none"/>
+                <location type="meta" path="/none"/>
+            </locations>
+        </cluster>
+    </clusters>
+    
+    <locations>
+        <location type="data" path="/user/ambari-qa/falcon/demo/primary/feed-import-filesystem/${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}"/>
+        <location type="stats" path="/none"/>
+        <location type="meta" path="/none"/>
+    </locations>
+    
+    <ACL owner="ambari-qa" group="users" permission="0755"/>
+    <schema location="/none" provider="none"/>
+
+   <lifecycle>
+        <retention-stage>
+            <frequency>days(1)</frequency>
+            <queue>default</queue>
+            <priority>LOW</priority>
+           <properties>
+                <property name="retention.policy.agebaseddelete.limit" value="hours(12)"></property>
+            </properties>
+        </retention-stage>
+    </lifecycle>
+</feed>

http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/generic-datasource.xml
----------------------------------------------------------------------
diff --git a/examples/entity/importexport/generic-datasource.xml b/examples/entity/importexport/generic-datasource.xml
new file mode 100755
index 0000000..74f15ca
--- /dev/null
+++ b/examples/entity/importexport/generic-datasource.xml
@@ -0,0 +1,49 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+
+<datasource colo="west-coast" description="Generic database on west coast" type="generic" name="generic-db" xmlns="uri:falcon:datasource:0.1">
+    <tags>owner=foobar@ambari.apache.org, consumer=phoe@ambari.apache.org</tags>
+    <interfaces>
+ 
+        <!-- ***** read interface ***** -->
+        <interface type="readonly" endpoint="jdbc:mysql://c6402/test">
+            <credential type="password-text">
+                <userName>sqoop_user</userName>
+                <passwordText>sqoop</passwordText>
+            </credential>
+        </interface>
+
+        <!-- ***** write interface ***** -->
+        <interface type="write"  endpoint="jdbc:mysql://c6402/test">
+            <credential type="password-text">
+                <userName>sqoop2_user</userName>
+                <passwordText>sqoop</passwordText>
+            </credential>
+        </interface>
+
+    </interfaces>
+
+    <driver>
+        <clazz>com.mysql.jdbc.Driver</clazz>
+        <jar>/user/oozie/share/lib/lib_20160628052542/sqoop/mysql-connector-java-5.1.31.jar</jar>
+    </driver>
+
+</datasource>
+

http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/generic-feed-import-filesystem.xml
----------------------------------------------------------------------
diff --git a/examples/entity/importexport/generic-feed-import-filesystem.xml b/examples/entity/importexport/generic-feed-import-filesystem.xml
new file mode 100755
index 0000000..01ccdab
--- /dev/null
+++ b/examples/entity/importexport/generic-feed-import-filesystem.xml
@@ -0,0 +1,72 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+
+<!--
+ A feed representing Hourly customer email data retained for 90 days
+ -->
+<feed description="Raw customer email feed" name="generic-feed-import-filesystem" xmlns="uri:falcon:feed:0.1">
+    <tags>externalSystem=USWestEmailServers,classification=secure</tags>
+    <groups>DataImportPipeline</groups>
+    <frequency>minutes(10)</frequency>
+    <late-arrival cut-off="hours(4)"/>
+    <clusters>
+        <cluster name="primaryCluster" type="source">
+            <validity start="2016-04-10T00:00Z" end="2017-03-31T00:00Z"/>
+            <retention limit="days(90)" action="delete"/>
+            <import> 
+                <source name="generic-db" tableName="db_raw_data">
+                    <extract type="full">
+                        <mergepolicy>snapshot</mergepolicy>
+                    </extract>    
+                    <fields>
+                           <includes>
+                             <field>id</field>
+                             <field>name</field>
+                           </includes>
+                    </fields>
+                </source>
+                <arguments>
+                    <argument name="--split-by" value="id"/>
+                    <argument name="--num-mappers" value="2"/>
+                </arguments>
+            </import>
+        </cluster>
+    </clusters>
+    
+    <locations>
+        <location type="data" path="/user/ambari-qa/falcon/demo/primary/generic-feed-import-filesystem/${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}"/>
+        <location type="stats" path="/none"/>
+        <location type="meta" path="/none"/>
+    </locations>
+    
+    <ACL owner="ambari-qa" group="users" permission="0755"/>
+    <schema location="/none" provider="none"/>
+
+   <lifecycle>
+        <retention-stage>
+            <frequency>days(1)</frequency>
+            <queue>default</queue>
+            <priority>LOW</priority>
+           <properties>
+                <property name="retention.policy.agebaseddelete.limit" value="hours(12)"></property>
+            </properties>
+        </retention-stage>
+    </lifecycle>
+</feed>

http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/mysql-datasource.xml
----------------------------------------------------------------------
diff --git a/examples/entity/importexport/mysql-datasource.xml b/examples/entity/importexport/mysql-datasource.xml
new file mode 100755
index 0000000..293b0bb
--- /dev/null
+++ b/examples/entity/importexport/mysql-datasource.xml
@@ -0,0 +1,58 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+
+<datasource colo="west-coast" description="MySQL database on west coast" type="mysql" name="mysql-db" xmlns="uri:falcon:datasource:0.1">
+    <tags>owner=foobar@ambari.apache.org, consumer=phoe@ambari.apache.org</tags>
+    <interfaces>
+ 
+        <!-- ***** read interface ***** -->
+        <interface type="readonly" endpoint="jdbc:mysql://c6402/test">
+            <credential type="password-text">
+                <userName>sqoop_user</userName>
+                <passwordText>sqoop</passwordText>
+            </credential>
+        </interface>
+
+        <!-- ***** write interface ***** -->
+        <interface type="write"  endpoint="jdbc:mysql://c6402/test">
+            <credential type="password-text">
+                <userName>sqoop2_user</userName>
+                <passwordText>sqoop</passwordText>
+            </credential>
+        </interface>
+
+        <!-- ***** default credential ***** -->
+        <credential type="password-alias">
+            <userName>sqoop2_user</userName>
+            <passwordAlias>
+                <alias>sqoop.password.alias</alias>
+                <providerPath>hdfs://c6407.ambari.apache.org:8020/user/ambari-qa/sqoop_password.jceks</providerPath>
+            </passwordAlias>
+        </credential>
+
+    </interfaces>
+
+    <driver>
+        <clazz>com.mysql.jdbc.Driver</clazz>
+        <jar>/user/oozie/share/lib/lib_20160628052542/sqoop/mysql-connector-java-5.1.31.jar</jar>
+    </driver>
+
+</datasource>
+

http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/mysql-feed-export-filesystem.xml
----------------------------------------------------------------------
diff --git a/examples/entity/importexport/mysql-feed-export-filesystem.xml b/examples/entity/importexport/mysql-feed-export-filesystem.xml
new file mode 100755
index 0000000..2e07aa2
--- /dev/null
+++ b/examples/entity/importexport/mysql-feed-export-filesystem.xml
@@ -0,0 +1,62 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->#
+
+<!--
+ A feed representing Hourly customer email data retained for 90 days
+ -->
+<feed description="Raw customer email feed" name="mysql-feed-export-filesystem" xmlns="uri:falcon:feed:0.1">
+    <tags>externalSystem=USWestEmailServers,classification=secure</tags>
+    <groups>DataImportPipeline</groups>
+    <frequency>minutes(10)</frequency>
+    <late-arrival cut-off="hours(4)"/>
+    <clusters>
+        <cluster name="primaryCluster" type="source">
+            <validity start="2016-04-10T00:00Z" end="2017-03-31T00:00Z"/>
+            <retention limit="days(90)" action="delete"/>
+            <export>
+                  <target name="mysql-db" tableName="db_export_fs">
+                      <load type="allowinsert"/>
+                  </target>    
+                  <arguments>
+                       <argument name="--update-key" value="id"/>
+                  </arguments>
+              </export>
+        </cluster>
+    </clusters>
+    
+    <locations>
+        <location type="data" path="/user/ambari-qa/falcon/demo/primary/mysql-feed-import-filesystem/${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}"/>
+        <location type="stats" path="/none"/>
+        <location type="meta" path="/none"/>
+    </locations>
+    
+    <ACL owner="ambari-qa" group="users" permission="0755"/>
+    <schema location="/none" provider="none"/>
+
+   <lifecycle>
+        <retention-stage>
+            <frequency>days(1)</frequency>
+            <queue>default</queue>
+            <priority>LOW</priority>
+           <properties>
+                <property name="retention.policy.agebaseddelete.limit" value="hours(12)"></property>
+            </properties>
+        </retention-stage>
+    </lifecycle>
+</feed>

http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/mysql-feed-import-filesystem.xml
----------------------------------------------------------------------
diff --git a/examples/entity/importexport/mysql-feed-import-filesystem.xml b/examples/entity/importexport/mysql-feed-import-filesystem.xml
new file mode 100755
index 0000000..b02b86f
--- /dev/null
+++ b/examples/entity/importexport/mysql-feed-import-filesystem.xml
@@ -0,0 +1,72 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+
+<!--
+ A feed representing Hourly customer email data retained for 90 days
+ -->
+<feed description="Raw customer email feed" name="mysql-feed-import-filesystem" xmlns="uri:falcon:feed:0.1">
+    <tags>externalSystem=USWestEmailServers,classification=secure</tags>
+    <groups>DataImportPipeline</groups>
+    <frequency>minutes(10)</frequency>
+    <late-arrival cut-off="hours(4)"/>
+    <clusters>
+        <cluster name="primaryCluster" type="source">
+            <validity start="2016-04-10T00:00Z" end="2017-03-31T00:00Z"/>
+            <retention limit="days(90)" action="delete"/>
+            <import> 
+                <source name="mysql-db" tableName="db_raw_data">
+                    <extract type="full">
+                        <mergepolicy>snapshot</mergepolicy>
+                    </extract>    
+                    <fields>
+                           <includes>
+                             <field>id</field>
+                             <field>name</field>
+                           </includes>
+                    </fields>
+                </source>
+                <arguments>
+                       <argument name="--split-by" value="id"/>
+                    <argument name="--num-mappers" value="2"/>
+                </arguments>
+            </import>
+        </cluster>
+    </clusters>
+    
+    <locations>
+        <location type="data" path="/user/ambari-qa/falcon/demo/primary/mysql-feed-import-filesystem/${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}"/>
+        <location type="stats" path="/none"/>
+        <location type="meta" path="/none"/>
+    </locations>
+    
+    <ACL owner="ambari-qa" group="users" permission="0755"/>
+    <schema location="/none" provider="none"/>
+
+   <lifecycle>
+        <retention-stage>
+            <frequency>days(1)</frequency>
+            <queue>default</queue>
+            <priority>LOW</priority>
+           <properties>
+                <property name="retention.policy.agebaseddelete.limit" value="hours(12)"></property>
+            </properties>
+        </retention-stage>
+    </lifecycle>
+</feed>

http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/netezza-datasource.xml
----------------------------------------------------------------------
diff --git a/examples/entity/importexport/netezza-datasource.xml b/examples/entity/importexport/netezza-datasource.xml
new file mode 100755
index 0000000..1e4dfe6
--- /dev/null
+++ b/examples/entity/importexport/netezza-datasource.xml
@@ -0,0 +1,57 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+
+<datasource colo="west-coast" description="Netezza database on west coast" type="netezza" name="netezza-db" xmlns="uri:falcon:datasource:0.1">
+    <tags>owner=foobar@ambari.apache.org, consumer=phoe@ambari.apache.org</tags>
+    <interfaces>
+ 
+        <!-- ***** read interface ***** -->
+        <interface type="readonly" endpoint="jdbc:netezza://10.10.20.14:5480/test">
+            <credential type="password-text">
+                <userName>admin</userName>
+        <passwordText>password</passwordText>
+            </credential>
+        </interface>
+
+        <!-- ***** write interface ***** -->
+        <interface type="write"  endpoint="jdbc:netezza://10.10.20.14:5480/test">
+            <credential type="password-text">
+                <userName>admin</userName>
+                <passwordText>password</passwordText>
+            </credential>
+        </interface>
+
+        <!-- ***** default credential ***** -->
+        <credential type="password-alias">
+           <userName>admin</userName>
+           <passwordAlias>
+               <alias>sqoop.password.alias</alias>
+               <providerPath>hdfs://c6401.ambari.apache.org:8020/user/ambari-qa/sqoop_password.jceks</providerPath>
+           </passwordAlias>
+        </credential>
+
+    </interfaces>
+
+    <driver>
+        <clazz>org.netezza.Driver</clazz>
+        <jar>/user/oozie/share/lib/lib_20160628052542/sqoop/nzjdbc.jar</jar>
+    </driver>
+</datasource>
+

http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/netezza-feed-import-filesystem.xml
----------------------------------------------------------------------
diff --git a/examples/entity/importexport/netezza-feed-import-filesystem.xml b/examples/entity/importexport/netezza-feed-import-filesystem.xml
new file mode 100755
index 0000000..bbbcbad
--- /dev/null
+++ b/examples/entity/importexport/netezza-feed-import-filesystem.xml
@@ -0,0 +1,72 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+
+<!--
+ A feed representing Hourly customer email data retained for 90 days
+ -->
+<feed description="Raw customer email feed" name="netezza-feed-import-filesystem" xmlns="uri:falcon:feed:0.1">
+    <tags>externalSystem=USWestEmailServers,classification=secure</tags>
+    <groups>DataImportPipeline</groups>
+    <frequency>minutes(10)</frequency>
+    <late-arrival cut-off="hours(4)"/>
+    <clusters>
+        <cluster name="primaryCluster" type="source">
+            <validity start="2016-04-10T00:00Z" end="2017-03-31T00:00Z"/>
+            <retention limit="days(90)" action="delete"/>
+            <import> 
+                <source name="netezza-db" tableName="db_raw_data">
+                    <extract type="full">
+                        <mergepolicy>snapshot</mergepolicy>
+                    </extract>    
+                    <fields>
+                           <includes>
+                             <field>id</field>
+                             <field>name</field>
+                           </includes>
+                    </fields>
+                </source>
+                <arguments>
+                    <argument name="--split-by" value="id"/>
+                    <argument name="--num-mappers" value="2"/>
+                </arguments>
+            </import>
+        </cluster>
+    </clusters>
+    
+    <locations>
+        <location type="data" path="/user/ambari-qa/falcon/demo/primary/netezza-feed-import-filesystem/${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}"/>
+        <location type="stats" path="/none"/>
+        <location type="meta" path="/none"/>
+    </locations>
+    
+    <ACL owner="ambari-qa" group="users" permission="0755"/>
+    <schema location="/none" provider="none"/>
+
+   <lifecycle>
+        <retention-stage>
+            <frequency>days(1)</frequency>
+            <queue>default</queue>
+            <priority>LOW</priority>
+           <properties>
+                <property name="retention.policy.agebaseddelete.limit" value="hours(12)"></property>
+            </properties>
+        </retention-stage>
+    </lifecycle>
+</feed>

http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/oracle-datasource.xml
----------------------------------------------------------------------
diff --git a/examples/entity/importexport/oracle-datasource.xml b/examples/entity/importexport/oracle-datasource.xml
new file mode 100755
index 0000000..535640c
--- /dev/null
+++ b/examples/entity/importexport/oracle-datasource.xml
@@ -0,0 +1,50 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+
+<datasource colo="west-coast" description="Oracle database on west coast" type="oracle" name="oracle-db" xmlns="uri:falcon:datasource:0.1">
+    <tags>owner=foobar@ambari.apache.org, consumer=phoe@ambari.apache.org</tags>
+    <interfaces>
+ 
+        <!-- ***** read interface ***** -->
+        <interface type="readonly" endpoint="jdbc:oracle:thin:@172.25.16.82:1521:XE">
+            <credential type="password-text">
+                <userName>SQOOP</userName>
+                <passwordText>sqoop</passwordText>
+            </credential>
+        </interface>
+
+        <!-- ***** write interface ***** -->
+        <interface type="write" endpoint="jdbc:oracle:thin:@172.25.16.82:1521:XE">
+            <credential type="password-text">
+                <userName>SQOOP</userName>
+                <passwordText>sqoop</passwordText>
+            </credential>
+        </interface>
+
+    </interfaces>
+
+    <driver>
+        <clazz>oracle.jdbc.OracleDriver</clazz>
+        <jar>/user/oozie/share/lib/lib_20160628052542/sqoop/ojdbc6.jar</jar>
+        <jar>/user/oozie/share/lib/lib_20160628052542/sqoop/orai18n.jar</jar>
+    </driver>
+
+</datasource>
+

http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/oracle-feed-import-filesystem.xml
----------------------------------------------------------------------
diff --git a/examples/entity/importexport/oracle-feed-import-filesystem.xml b/examples/entity/importexport/oracle-feed-import-filesystem.xml
new file mode 100755
index 0000000..5bba3de
--- /dev/null
+++ b/examples/entity/importexport/oracle-feed-import-filesystem.xml
@@ -0,0 +1,72 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+
+<!--
+ A feed representing Hourly customer email data retained for 90 days
+ -->
+<feed description="Raw customer email feed" name="oracle-feed-import-filesystem" xmlns="uri:falcon:feed:0.1">
+    <tags>externalSystem=USWestEmailServers,classification=secure</tags>
+    <groups>DataImportPipeline</groups>
+    <frequency>minutes(10)</frequency>
+    <late-arrival cut-off="hours(4)"/>
+    <clusters>
+        <cluster name="primaryCluster" type="source">
+            <validity start="2016-04-10T00:00Z" end="2017-03-31T00:00Z"/>
+            <retention limit="days(90)" action="delete"/>
+            <import> 
+                <source name="oracle-db" tableName="DB_RAW_DATA">
+                    <extract type="full">
+                        <mergepolicy>snapshot</mergepolicy>
+                    </extract>    
+                    <fields>
+                           <includes>
+                             <field>ID</field>
+                             <field>NAME</field>
+                           </includes>
+                    </fields>
+                </source>
+                <arguments>
+                       <argument name="--split-by" value="ID"/>
+                    <argument name="--num-mappers" value="2"/>
+                </arguments>
+            </import>
+        </cluster>
+    </clusters>
+    
+    <locations>
+        <location type="data" path="/user/ambari-qa/falcon/demo/primary/oracle-feed-import-filesystem/${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}"/>
+        <location type="stats" path="/none"/>
+        <location type="meta" path="/none"/>
+    </locations>
+    
+    <ACL owner="ambari-qa" group="users" permission="0755"/>
+    <schema location="/none" provider="none"/>
+
+   <lifecycle>
+        <retention-stage>
+            <frequency>days(1)</frequency>
+            <queue>default</queue>
+            <priority>LOW</priority>
+           <properties>
+                <property name="retention.policy.agebaseddelete.limit" value="hours(12)"></property>
+            </properties>
+        </retention-stage>
+    </lifecycle>
+</feed>

http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/pgsql-datasource.xml
----------------------------------------------------------------------
diff --git a/examples/entity/importexport/pgsql-datasource.xml b/examples/entity/importexport/pgsql-datasource.xml
new file mode 100755
index 0000000..b161bd3
--- /dev/null
+++ b/examples/entity/importexport/pgsql-datasource.xml
@@ -0,0 +1,55 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+
+<datasource colo="west-coast" description="Postgres database on west coast" type="postgres" name="pgsql-db" xmlns="uri:falcon:datasource:0.1">
+    <tags>owner=foobar@ambari.apache.org, consumer=phoe@ambari.apache.org</tags>
+    <interfaces>
+        <!-- ***** read interface ***** -->
+        <interface type="readonly" endpoint="jdbc:postgresql://c6402:5432/test">
+            <credential type="password-text">
+                <userName>ambari-qa</userName>
+                <passwordText>sqoop</passwordText>
+            </credential>
+        </interface>
+
+        <!-- ***** write interface ***** -->
+        <interface type="write"  endpoint="jdbc:postgresql://c6402:5432/test">
+            <credential type="password-text">
+                <userName>ambari-qa</userName>
+                <passwordText>sqoop</passwordText>
+            </credential>
+        </interface>
+
+        <!-- ***** default credential ***** -->
+        <credential type="password-alias">
+            <userName>ambari-qa</userName>
+            <passwordAlias>
+                <alias>sqoop.password.alias</alias>
+                <providerPath>hdfs://c6401.ambari.apache.org:8020/user/ambari-qa/sqoop_password.jceks</providerPath>
+            </passwordAlias>
+        </credential>
+    </interfaces>
+
+    <driver>
+        <clazz>org.postgresql.Driver</clazz>
+        <jar>/user/oozie/share/lib/lib_20160628052542/sqoop/postgresql-9.4.1208.jar</jar>
+    </driver>
+</datasource>
+

http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/pgsql-feed-import-filesystem.xml
----------------------------------------------------------------------
diff --git a/examples/entity/importexport/pgsql-feed-import-filesystem.xml b/examples/entity/importexport/pgsql-feed-import-filesystem.xml
new file mode 100755
index 0000000..fa88cd1
--- /dev/null
+++ b/examples/entity/importexport/pgsql-feed-import-filesystem.xml
@@ -0,0 +1,70 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+<!--
+ A feed representing Hourly customer email data retained for 90 days
+ -->
+<feed description="Raw customer email feed" name="pgsql-feed-import-filesystem" xmlns="uri:falcon:feed:0.1">
+    <tags>externalSystem=USWestEmailServers,classification=secure</tags>
+    <groups>DataImportPipeline</groups>
+    <frequency>minutes(10)</frequency>
+    <late-arrival cut-off="hours(4)"/>
+    <clusters>
+        <cluster name="primaryCluster" type="source">
+            <validity start="2016-04-10T00:00Z" end="2017-03-31T00:00Z"/>
+            <retention limit="days(90)" action="delete"/>
+            <import> 
+                <source name="pgsql-db" tableName="db_raw_data">
+                    <extract type="full">
+                        <mergepolicy>snapshot</mergepolicy>
+                    </extract>    
+                    <fields>
+                           <includes>
+                             <field>id</field>
+                             <field>name</field>
+                           </includes>
+                    </fields>
+                </source>
+                <arguments>
+                       <argument name="--split-by" value="id"/>
+                    <argument name="--num-mappers" value="2"/>
+                </arguments>
+            </import>
+        </cluster>
+    </clusters>
+    
+    <locations>
+        <location type="data" path="/user/ambari-qa/falcon/demo/primary/pgsql-feed-import-filesystem/${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}"/>
+        <location type="stats" path="/none"/>
+        <location type="meta" path="/none"/>
+    </locations>
+    
+    <ACL owner="ambari-qa" group="users" permission="0755"/>
+    <schema location="/none" provider="none"/>
+
+   <lifecycle>
+        <retention-stage>
+            <frequency>days(1)</frequency>
+            <queue>default</queue>
+            <priority>LOW</priority>
+           <properties>
+                <property name="retention.policy.agebaseddelete.limit" value="hours(12)"></property>
+            </properties>
+        </retention-stage>
+    </lifecycle>
+</feed>

http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/primary_cluster.xml
----------------------------------------------------------------------
diff --git a/examples/entity/importexport/primary_cluster.xml b/examples/entity/importexport/primary_cluster.xml
new file mode 100755
index 0000000..4d43fd7
--- /dev/null
+++ b/examples/entity/importexport/primary_cluster.xml
@@ -0,0 +1,33 @@
+<?xml version="1.0"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+<cluster colo="USWestOregon" description="oregonHadoopCluster" name="primaryCluster" xmlns="uri:falcon:cluster:0.1">
+    <interfaces>
+        <interface type="readonly"  endpoint="webhdfs://localhost:50070" version="2.2.0" />
+        <interface type="write"     endpoint="hdfs://localhost:8020" version="2.2.0" />
+        <interface type="execute"   endpoint="localhost:8050" version="2.2.0" />
+        <interface type="workflow"  endpoint="http://localhost:11000/oozie/" version="4.0.0" />
+        <interface type="messaging" endpoint="tcp://localhost:61616?daemon=true" version="5.1.6" />
+        <interface type="registry"  endpoint="thrift://localhost:9083" version="0.14.0" />
+    </interfaces>
+    <locations>
+        <location name="staging" path="/apps/falcon/primaryCluster/staging" />
+        <location name="temp" path="/tmp" />
+        <location name="working" path="/apps/falcon/primaryCluster/working" />
+    </locations>
+</cluster>

http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/td-datasource.xml
----------------------------------------------------------------------
diff --git a/examples/entity/importexport/td-datasource.xml b/examples/entity/importexport/td-datasource.xml
new file mode 100755
index 0000000..a9eb4ed
--- /dev/null
+++ b/examples/entity/importexport/td-datasource.xml
@@ -0,0 +1,56 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+<datasource colo="west-coast" description="Teradata database on west coast" type="teradata" name="td-db" xmlns="uri:falcon:datasource:0.1">
+    <tags>owner=foobar@ambari.apache.org, consumer=phoe@ambari.apache.org</tags>
+    <interfaces>
+ 
+        <!-- ***** read interface ***** -->
+        <interface type="readonly" endpoint="jdbc:teradata://10.10.20.13/DATABASE=retail">
+            <credential type="password-text">
+                <userName>dbc</userName>
+        <passwordText>dbc</passwordText>
+            </credential>
+        </interface>
+
+        <!-- ***** write interface ***** -->
+        <interface type="write"  endpoint="jdbc:teradata://10.10.20.13/DATABASE=retail">
+            <credential type="password-text">
+                <userName>dbc</userName>
+                <passwordText>dbc</passwordText>
+            </credential>
+        </interface>
+
+        <!-- ***** default credential ***** -->
+        <credential type="password-alias">
+            <userName>dbc</userName>
+            <passwordAlias>
+                <alias>sqoop.password.alias</alias>
+                <providerPath>hdfs://c6401.ambari.apache.org:8020/user/ambari-qa/sqoop_password.jceks</providerPath>
+            </passwordAlias>
+        </credential>
+
+    </interfaces>
+
+    <driver>
+        <clazz>com.teradata.jdbc.TeraDriver</clazz>
+        <jar>/user/oozie/share/lib/lib_20160628052542/sqoop/terajdbc4.jar</jar>
+        <jar>/user/oozie/share/lib/lib_20160628052542/sqoop/tdgssconfig.jar</jar>
+    </driver>
+</datasource>
+

http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/examples/entity/importexport/td-feed-import-filesystem.xml
----------------------------------------------------------------------
diff --git a/examples/entity/importexport/td-feed-import-filesystem.xml b/examples/entity/importexport/td-feed-import-filesystem.xml
new file mode 100755
index 0000000..6dd94ed
--- /dev/null
+++ b/examples/entity/importexport/td-feed-import-filesystem.xml
@@ -0,0 +1,70 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+<!--
+ A feed representing Hourly customer email data retained for 90 days
+ -->
+<feed description="Raw customer email feed" name="td-feed-import-filesystem" xmlns="uri:falcon:feed:0.1">
+    <tags>externalSystem=USWestEmailServers,classification=secure</tags>
+    <groups>DataImportPipeline</groups>
+    <frequency>minutes(10)</frequency>
+    <late-arrival cut-off="hours(4)"/>
+    <clusters>
+        <cluster name="primaryCluster" type="source">
+            <validity start="2016-04-10T00:00Z" end="2017-03-31T00:00Z"/>
+            <retention limit="days(90)" action="delete"/>
+            <import> 
+                <source name="td-db" tableName="db_raw_data">
+                    <extract type="full">
+                        <mergepolicy>snapshot</mergepolicy>
+                    </extract>    
+                    <fields>
+                           <includes>
+                             <field>id</field>
+                             <field>name</field>
+                           </includes>
+                    </fields>
+                </source>
+                <arguments>
+                    <argument name="--split-by" value="id"/>
+                    <argument name="--num-mappers" value="2"/>
+                </arguments>
+            </import>
+        </cluster>
+    </clusters>
+    
+    <locations>
+        <location type="data" path="/user/ambari-qa/falcon/demo/primary/td-feed-import-filesystem/${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}"/>
+        <location type="stats" path="/none"/>
+        <location type="meta" path="/none"/>
+    </locations>
+    
+    <ACL owner="ambari-qa" group="users" permission="0755"/>
+    <schema location="/none" provider="none"/>
+
+   <lifecycle>
+        <retention-stage>
+            <frequency>days(1)</frequency>
+            <queue>default</queue>
+            <priority>LOW</priority>
+           <properties>
+                <property name="retention.policy.agebaseddelete.limit" value="hours(12)"></property>
+            </properties>
+        </retention-stage>
+    </lifecycle>
+</feed>

http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/oozie/src/main/java/org/apache/falcon/oozie/DatabaseExportWorkflowBuilder.java
----------------------------------------------------------------------
diff --git a/oozie/src/main/java/org/apache/falcon/oozie/DatabaseExportWorkflowBuilder.java b/oozie/src/main/java/org/apache/falcon/oozie/DatabaseExportWorkflowBuilder.java
index e7ca869..93f3d1f 100644
--- a/oozie/src/main/java/org/apache/falcon/oozie/DatabaseExportWorkflowBuilder.java
+++ b/oozie/src/main/java/org/apache/falcon/oozie/DatabaseExportWorkflowBuilder.java
@@ -94,12 +94,15 @@ public class DatabaseExportWorkflowBuilder extends ExportWorkflowBuilder {
         StringBuilder sqoopArgs = new StringBuilder();
         StringBuilder sqoopOptions = new StringBuilder();
 
-        buildConnectArg(sqoopArgs, cluster).append(ImportExportCommon.ARG_SEPARATOR);
-        buildTableArg(sqoopArgs, cluster).append(ImportExportCommon.ARG_SEPARATOR);
-        Datasource datasource = DatasourceHelper.getDatasource(FeedHelper.getExportDatasourceName(
-                FeedHelper.getCluster(entity, cluster.getName())));
-        ImportExportCommon.buildUserPasswordArg(sqoopArgs, sqoopOptions, datasource)
-                .append(ImportExportCommon.ARG_SEPARATOR);
+        org.apache.falcon.entity.v0.feed.Cluster feedCluster = FeedHelper.getCluster(entity, cluster.getName());
+        Datasource datasource = DatasourceHelper.getDatasource(FeedHelper.getExportDatasourceName(feedCluster));
+        ImportExportCommon.buildDriverArgs(sqoopArgs, datasource).append(ImportExportCommon.ARG_SEPARATOR);
+        ImportExportCommon.buildConnectArg(sqoopArgs, DatasourceHelper.getWriteEndpoint(datasource))
+            .append(ImportExportCommon.ARG_SEPARATOR);
+        ImportExportCommon.buildTableArg(sqoopArgs, FeedHelper.getExportDataSourceTableName(feedCluster))
+            .append(ImportExportCommon.ARG_SEPARATOR);
+        ImportExportCommon.buildUserPasswordArg(sqoopArgs, sqoopOptions,
+            DatasourceHelper.getWritePasswordInfo(datasource)).append(ImportExportCommon.ARG_SEPARATOR);
         buildNumMappers(sqoopArgs, extraArgs).append(ImportExportCommon.ARG_SEPARATOR);
         buildArguments(sqoopArgs, extraArgs, feed, cluster).append(ImportExportCommon.ARG_SEPARATOR);
         buildLoadType(sqoopArgs, cluster).append(ImportExportCommon.ARG_SEPARATOR);
@@ -111,19 +114,6 @@ public class DatabaseExportWorkflowBuilder extends ExportWorkflowBuilder {
                 .append(sqoopArgs).toString();
     }
 
-    private StringBuilder buildConnectArg(StringBuilder builder, Cluster cluster) throws FalconException {
-        org.apache.falcon.entity.v0.feed.Cluster feedCluster = FeedHelper.getCluster(entity, cluster.getName());
-        return builder.append("--connect").append(ImportExportCommon.ARG_SEPARATOR)
-                .append(DatasourceHelper.getReadOnlyEndpoint(
-                        DatasourceHelper.getDatasource(FeedHelper.getExportDatasourceName(feedCluster))));
-    }
-
-    private StringBuilder buildTableArg(StringBuilder builder, Cluster cluster) throws FalconException {
-        org.apache.falcon.entity.v0.feed.Cluster feedCluster = FeedHelper.getCluster(entity, cluster.getName());
-        return builder.append("--table").append(ImportExportCommon.ARG_SEPARATOR)
-                .append(FeedHelper.getExportDataSourceTableName(feedCluster));
-    }
-
     private StringBuilder buildLoadType(StringBuilder builder, Cluster cluster)
         throws FalconException {
         org.apache.falcon.entity.v0.feed.Cluster feedCluster = FeedHelper.getCluster(entity, cluster.getName());

http://git-wip-us.apache.org/repos/asf/falcon/blob/292dfed3/oozie/src/main/java/org/apache/falcon/oozie/DatabaseImportWorkflowBuilder.java
----------------------------------------------------------------------
diff --git a/oozie/src/main/java/org/apache/falcon/oozie/DatabaseImportWorkflowBuilder.java b/oozie/src/main/java/org/apache/falcon/oozie/DatabaseImportWorkflowBuilder.java
index 1efa368..77d5462 100644
--- a/oozie/src/main/java/org/apache/falcon/oozie/DatabaseImportWorkflowBuilder.java
+++ b/oozie/src/main/java/org/apache/falcon/oozie/DatabaseImportWorkflowBuilder.java
@@ -95,13 +95,16 @@ public class DatabaseImportWorkflowBuilder extends ImportWorkflowBuilder {
         Map<String, String> extraArgs = getArguments(cluster);
         StringBuilder sqoopArgs = new StringBuilder();
         StringBuilder sqoopOptions = new StringBuilder();
-        buildDriverArgs(sqoopArgs, cluster).append(ImportExportCommon.ARG_SEPARATOR);
-        buildConnectArg(sqoopArgs, cluster).append(ImportExportCommon.ARG_SEPARATOR);
-        buildTableArg(sqoopArgs, cluster).append(ImportExportCommon.ARG_SEPARATOR);
-        Datasource datasource = DatasourceHelper.getDatasource(FeedHelper.getImportDatasourceName(
-            FeedHelper.getCluster(entity, cluster.getName())));
-        ImportExportCommon.buildUserPasswordArg(sqoopArgs, sqoopOptions, datasource)
+
+        org.apache.falcon.entity.v0.feed.Cluster feedCluster = FeedHelper.getCluster(entity, cluster.getName());
+        Datasource datasource = DatasourceHelper.getDatasource(FeedHelper.getImportDatasourceName(feedCluster));
+        ImportExportCommon.buildDriverArgs(sqoopArgs, datasource).append(ImportExportCommon.ARG_SEPARATOR);
+        ImportExportCommon.buildConnectArg(sqoopArgs, DatasourceHelper.getReadOnlyEndpoint(datasource))
             .append(ImportExportCommon.ARG_SEPARATOR);
+        ImportExportCommon.buildTableArg(sqoopArgs, FeedHelper.getImportDataSourceTableName(feedCluster))
+            .append(ImportExportCommon.ARG_SEPARATOR);
+        ImportExportCommon.buildUserPasswordArg(sqoopArgs, sqoopOptions,
+            DatasourceHelper.getReadPasswordInfo(datasource)).append(ImportExportCommon.ARG_SEPARATOR);
         buildNumMappers(sqoopArgs, extraArgs).append(ImportExportCommon.ARG_SEPARATOR);
         buildArguments(sqoopArgs, extraArgs).append(ImportExportCommon.ARG_SEPARATOR);
         buildTargetArg(sqoopArgs, feed, cluster).append(ImportExportCommon.ARG_SEPARATOR);
@@ -112,28 +115,6 @@ public class DatabaseImportWorkflowBuilder extends ImportWorkflowBuilder {
             .append(sqoopArgs).toString();
     }
 
-    private StringBuilder buildDriverArgs(StringBuilder builder, Cluster cluster) throws FalconException {
-        org.apache.falcon.entity.v0.feed.Cluster feedCluster = FeedHelper.getCluster(entity, cluster.getName());
-        Datasource db = DatasourceHelper.getDatasource(FeedHelper.getImportDatasourceName(feedCluster));
-        if ((db.getDriver() != null) && (db.getDriver().getClazz() != null)) {
-            builder.append("--driver").append(ImportExportCommon.ARG_SEPARATOR).append(db.getDriver().getClazz());
-        }
-        return builder;
-    }
-
-    private StringBuilder buildConnectArg(StringBuilder builder, Cluster cluster) throws FalconException {
-        org.apache.falcon.entity.v0.feed.Cluster feedCluster = FeedHelper.getCluster(entity, cluster.getName());
-        return builder.append("--connect").append(ImportExportCommon.ARG_SEPARATOR)
-            .append(DatasourceHelper.getReadOnlyEndpoint(
-                DatasourceHelper.getDatasource(FeedHelper.getImportDatasourceName(feedCluster))));
-    }
-
-    private StringBuilder buildTableArg(StringBuilder builder, Cluster cluster) throws FalconException {
-        org.apache.falcon.entity.v0.feed.Cluster feedCluster = FeedHelper.getCluster(entity, cluster.getName());
-        return builder.append("--table").append(ImportExportCommon.ARG_SEPARATOR)
-            .append(FeedHelper.getImportDataSourceTableName(feedCluster));
-    }
-
     private StringBuilder buildTargetArg(StringBuilder builder, Feed feed, Cluster cluster)
         throws FalconException {
         Storage.TYPE feedStorageType = FeedHelper.getStorageType(feed, cluster);


Mime
View raw message