falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From shweth...@apache.org
Subject [1/9] FALCON-369 Refactor workflow builder. Contributed by Shwetha GS
Date Thu, 10 Jul 2014 06:57:30 GMT
Repository: incubator-falcon
Updated Branches:
  refs/heads/master 57953f77b -> 185b58885


http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/185b5888/src/main/examples/entity/filesystem/replication-feed.xml
----------------------------------------------------------------------
diff --git a/src/main/examples/entity/filesystem/replication-feed.xml b/src/main/examples/entity/filesystem/replication-feed.xml
new file mode 100644
index 0000000..ccf5cca
--- /dev/null
+++ b/src/main/examples/entity/filesystem/replication-feed.xml
@@ -0,0 +1,46 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+<feed description="replication feed" name="repl-feed" xmlns="uri:falcon:feed:0.1">
+    <groups>input</groups>
+
+    <frequency>minutes(1)</frequency>
+    <timezone>UTC</timezone>
+    <late-arrival cut-off="hours(1)"/>
+
+    <clusters>
+        <cluster name="local" type="source">
+            <validity start="2013-01-01T00:00Z" end="2030-01-01T00:00Z"/>
+            <retention limit="hours(2)" action="delete"/>
+        </cluster>
+        <cluster name="local-target" type="target">
+            <validity start="2013-11-15T00:00Z" end="2030-01-01T00:00Z"/>
+            <retention limit="hours(2)" action="delete"/>
+                <locations>
+                    <location type="data" path="/data/repl-in/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}"/>
+                </locations>
+        </cluster>
+    </clusters>
+
+    <locations>
+        <location type="data" path="/data/in/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}"/>
+    </locations>
+
+    <ACL owner="testuser" group="group" permission="0x644"/>
+    <schema location="/schema/log/log.format.csv" provider="csv"/>
+</feed>

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/185b5888/src/main/examples/entity/filesystem/standalone-cluster.xml
----------------------------------------------------------------------
diff --git a/src/main/examples/entity/filesystem/standalone-cluster.xml b/src/main/examples/entity/filesystem/standalone-cluster.xml
index 3c3c9f1..6fe4df3 100644
--- a/src/main/examples/entity/filesystem/standalone-cluster.xml
+++ b/src/main/examples/entity/filesystem/standalone-cluster.xml
@@ -23,7 +23,7 @@
 -->
 <cluster colo="local" description="Standalone cluster" name="local" xmlns="uri:falcon:cluster:0.1">
     <interfaces>
-        <interface type="readonly" endpoint="hftp://localhost:50010" version="1.1.2"/>
+        <interface type="readonly" endpoint="hdfs://localhost:8020" version="1.1.2"/>
 
         <interface type="write" endpoint="hdfs://localhost:8020" version="1.1.2"/>
 

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/185b5888/src/main/examples/entity/filesystem/standalone-target-cluster.xml
----------------------------------------------------------------------
diff --git a/src/main/examples/entity/filesystem/standalone-target-cluster.xml b/src/main/examples/entity/filesystem/standalone-target-cluster.xml
new file mode 100644
index 0000000..442449d
--- /dev/null
+++ b/src/main/examples/entity/filesystem/standalone-target-cluster.xml
@@ -0,0 +1,43 @@
+<?xml version="1.0"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+
+<!--
+   Example cluster definition that can be used with falcon and standalone 
+   hadoop and oozie instances
+-->
+<cluster colo="local" description="Standalone cluster" name="local-target" xmlns="uri:falcon:cluster:0.1">
+    <interfaces>
+        <interface type="readonly" endpoint="hdfs://localhost:8020" version="1.1.2"/>
+
+        <interface type="write" endpoint="hdfs://localhost:8020" version="1.1.2"/>
+
+        <interface type="execute" endpoint="localhost:8021" version="1.1.2"/>
+
+        <interface type="workflow" endpoint="http://localhost:11000/oozie/" version="4.0.0"/>
+
+        <interface type="messaging" endpoint="tcp://localhost:61616?daemon=true"  version="5.4.3"/>
+    </interfaces>
+    <locations>
+        <location name="staging" path="/projects/falcon/staging-target"/>
+        <location name="temp" path="/projects/falcon/tmp-target"/>
+        <location name="working" path="/projects/falcon/working-target"/>
+    </locations>
+    <properties>
+    </properties>
+</cluster>

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/185b5888/src/main/examples/entity/hcat/hcat-in-feed.xml
----------------------------------------------------------------------
diff --git a/src/main/examples/entity/hcat/hcat-in-feed.xml b/src/main/examples/entity/hcat/hcat-in-feed.xml
index 77f70db..5b2c998 100644
--- a/src/main/examples/entity/hcat/hcat-in-feed.xml
+++ b/src/main/examples/entity/hcat/hcat-in-feed.xml
@@ -16,7 +16,7 @@
   See the License for the specific language governing permissions and
   limitations under the License.
   -->
-<feed description="input" name="in" xmlns="uri:falcon:feed:0.1">
+<feed description="input" name="hcat-in" xmlns="uri:falcon:feed:0.1">
     <groups>input</groups>
 
     <frequency>minutes(1)</frequency>
@@ -24,7 +24,7 @@
     <late-arrival cut-off="hours(1)"/>
 
     <clusters>
-        <cluster name="local">
+        <cluster name="hcat-local">
             <validity start="2013-01-01T00:00Z" end="2030-01-01T00:00Z"/>
             <retention limit="hours(2)" action="delete"/>
         </cluster>

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/185b5888/src/main/examples/entity/hcat/hcat-out-feed.xml
----------------------------------------------------------------------
diff --git a/src/main/examples/entity/hcat/hcat-out-feed.xml b/src/main/examples/entity/hcat/hcat-out-feed.xml
index f09b2ed..26d1f18 100644
--- a/src/main/examples/entity/hcat/hcat-out-feed.xml
+++ b/src/main/examples/entity/hcat/hcat-out-feed.xml
@@ -16,7 +16,7 @@
   See the License for the specific language governing permissions and
   limitations under the License.
   -->
-<feed description="output" name="out" xmlns="uri:falcon:feed:0.1">
+<feed description="output" name="hcat-out" xmlns="uri:falcon:feed:0.1">
     <groups>output</groups>
 
     <frequency>minutes(5)</frequency>
@@ -24,7 +24,7 @@
     <late-arrival cut-off="hours(1)"/>
 
     <clusters>
-        <cluster name="local">
+        <cluster name="hcat-local">
             <validity start="2013-01-01T00:00Z" end="2030-01-01T00:00Z"/>
             <retention limit="hours(2)" action="delete"/>
         </cluster>

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/185b5888/src/main/examples/entity/hcat/hcat-pig-process.xml
----------------------------------------------------------------------
diff --git a/src/main/examples/entity/hcat/hcat-pig-process.xml b/src/main/examples/entity/hcat/hcat-pig-process.xml
index 753e5b0..6ee17eb 100644
--- a/src/main/examples/entity/hcat/hcat-pig-process.xml
+++ b/src/main/examples/entity/hcat/hcat-pig-process.xml
@@ -19,7 +19,7 @@
 
 <process name="hcat-pig-process" xmlns="uri:falcon:process:0.1">
     <clusters>
-        <cluster name="local">
+        <cluster name="hcat-local">
             <validity start="2013-11-15T00:05Z" end="2013-11-15T01:05Z"/>
         </cluster>
     </clusters>
@@ -30,13 +30,13 @@
     <timezone>UTC</timezone>
 
     <inputs>
-        <!-- In the pig script, the input paths will be available in a variable 'inpaths'
-->
-        <input name="input" feed="in" start="now(0,-5)" end="now(0,-1)"/>
+        <!-- In the pig script, the input paths will be available in a variable 'inparts'
-->
+        <input name="inparts" feed="in" start="now(0,-5)" end="now(0,-1)"/>
     </inputs>
 
     <outputs>
-        <!-- In the pig script, the output path will be available in a variable 'outpath'
-->
-        <output name="output" feed="out" instance="now(0,0)"/>
+        <!-- In the pig script, the output path will be available in a variable 'outparts'
-->
+        <output name="outpart" feed="out" instance="now(0,0)"/>
     </outputs>
 
     <workflow engine="pig" path="/app/pig/hcat-wordcount.pig"/>

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/185b5888/src/main/examples/entity/hcat/hcat-replication-feed.xml
----------------------------------------------------------------------
diff --git a/src/main/examples/entity/hcat/hcat-replication-feed.xml b/src/main/examples/entity/hcat/hcat-replication-feed.xml
new file mode 100644
index 0000000..2b43d28
--- /dev/null
+++ b/src/main/examples/entity/hcat/hcat-replication-feed.xml
@@ -0,0 +1,42 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+<feed description="input" name="hcat-repl-feed" xmlns="uri:falcon:feed:0.1">
+    <groups>input</groups>
+
+    <frequency>minutes(1)</frequency>
+    <timezone>UTC</timezone>
+    <late-arrival cut-off="hours(1)"/>
+
+    <clusters>
+        <cluster name="hcat-local" type="source">
+            <validity start="2013-01-01T00:00Z" end="2030-01-01T00:00Z"/>
+            <retention limit="hours(2)" action="delete"/>
+        </cluster>
+        <cluster name="hcat-local-target" type="target">
+            <validity start="2013-11-15T00:00Z" end="2030-01-01T00:00Z"/>
+            <retention limit="hours(2)" action="delete"/>
+            <table uri="catalog:default:repl_in_table#ds=${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}"
/>
+        </cluster>
+    </clusters>
+
+    <table uri="catalog:default:in_table#ds=${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}"
/>
+
+    <ACL owner="testuser" group="group" permission="0x644"/>
+    <schema location="/schema/log/log.format.csv" provider="csv"/>
+</feed>

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/185b5888/src/main/examples/entity/hcat/hcat-standalone-cluster.xml
----------------------------------------------------------------------
diff --git a/src/main/examples/entity/hcat/hcat-standalone-cluster.xml b/src/main/examples/entity/hcat/hcat-standalone-cluster.xml
index 30f0cb1..7962b1f 100644
--- a/src/main/examples/entity/hcat/hcat-standalone-cluster.xml
+++ b/src/main/examples/entity/hcat/hcat-standalone-cluster.xml
@@ -21,7 +21,7 @@
    Example cluster definition that can be used with falcon and standalone 
    hadoop and oozie instances
 -->
-<cluster colo="local" description="Standalone cluster with hacatalog" name="local" xmlns="uri:falcon:cluster:0.1">
+<cluster colo="local" description="Standalone cluster with hacatalog" name="hcat-local"
xmlns="uri:falcon:cluster:0.1">
     <interfaces>
         <interface type="readonly" endpoint="hftp://localhost:50010" version="1.1.2"/>
 
@@ -36,9 +36,9 @@
         <interface type="registry" endpoint="thrift://localhost:12000" version="0.11.0"/>
     </interfaces>
     <locations>
-        <location name="staging" path="/projects/falcon/staging"/>
-        <location name="temp" path="/projects/falcon/tmp"/>
-        <location name="working" path="/projects/falcon/working"/>
+        <location name="staging" path="/projects/falcon/hcat-staging"/>
+        <location name="temp" path="/projects/falcon/hcat-tmp"/>
+        <location name="working" path="/projects/falcon/hcat-working"/>
     </locations>
     <properties>
     </properties>

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/185b5888/src/main/examples/entity/hcat/hcat-standalone-target-cluster.xml
----------------------------------------------------------------------
diff --git a/src/main/examples/entity/hcat/hcat-standalone-target-cluster.xml b/src/main/examples/entity/hcat/hcat-standalone-target-cluster.xml
new file mode 100644
index 0000000..8de78cf
--- /dev/null
+++ b/src/main/examples/entity/hcat/hcat-standalone-target-cluster.xml
@@ -0,0 +1,45 @@
+<?xml version="1.0"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+
+<!--
+   Example cluster definition that can be used with falcon and standalone 
+   hadoop and oozie instances
+-->
+<cluster colo="hcat-local-target" description="Standalone cluster with hacatalog" name="hcat-local-target"
xmlns="uri:falcon:cluster:0.1">
+    <interfaces>
+        <interface type="readonly" endpoint="hftp://localhost:50010" version="1.1.2"/>
+
+        <interface type="write" endpoint="hdfs://localhost:8020" version="1.1.2"/>
+
+        <interface type="execute" endpoint="localhost:8021" version="1.1.2"/>
+
+        <interface type="workflow" endpoint="http://localhost:11000/oozie/" version="4.0.0"/>
+
+        <interface type="messaging" endpoint="tcp://localhost:61616?daemon=true"  version="5.4.3"/>
+
+        <interface type="registry" endpoint="thrift://localhost:12000" version="0.11.0"/>
+    </interfaces>
+    <locations>
+        <location name="staging" path="/projects/falcon/hcat-staging-target"/>
+        <location name="temp" path="/projects/falcon/hcat-tmp-target"/>
+        <location name="working" path="/projects/falcon/hcat-working-target"/>
+    </locations>
+    <properties>
+    </properties>
+</cluster>

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/185b5888/src/main/examples/entity/hcat/hive-process.xml
----------------------------------------------------------------------
diff --git a/src/main/examples/entity/hcat/hive-process.xml b/src/main/examples/entity/hcat/hive-process.xml
index 0f3c540..ad8b219 100644
--- a/src/main/examples/entity/hcat/hive-process.xml
+++ b/src/main/examples/entity/hcat/hive-process.xml
@@ -19,7 +19,7 @@
 
 <process name="hive-process" xmlns="uri:falcon:process:0.1">
     <clusters>
-        <cluster name="local">
+        <cluster name="hcat-local">
             <validity start="2013-11-15T00:05Z" end="2013-11-15T01:05Z"/>
         </cluster>
     </clusters>
@@ -30,13 +30,13 @@
     <timezone>UTC</timezone>
 
     <inputs>
-        <!-- In the pig script, the input paths will be available in a variable 'inpaths'
-->
-        <input name="input" feed="in" start="now(0,-5)" end="now(0,-1)"/>
+        <!-- In the pig script, the input paths will be available in a variable 'inparts'
-->
+        <input name="inparts" feed="hcat-in" start="now(0,-5)" end="now(0,-1)"/>
     </inputs>
 
     <outputs>
-        <!-- In the pig script, the output path will be available in a variable 'outpath'
-->
-        <output name="output" feed="out" instance="now(0,0)"/>
+        <!-- In the pig script, the output path will be available in a variable 'outpart'
-->
+        <output name="outpart" feed="hcat-out" instance="now(0,0)"/>
     </outputs>
 
     <workflow engine="hive" path="/app/hive/wordcount.hql"/>

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/185b5888/webapp/pom.xml
----------------------------------------------------------------------
diff --git a/webapp/pom.xml b/webapp/pom.xml
index d1e1ec1..9a5050e 100644
--- a/webapp/pom.xml
+++ b/webapp/pom.xml
@@ -404,6 +404,7 @@
                 <artifactId>maven-jetty-plugin</artifactId>
                 <version>${jetty.version}</version>
                 <configuration>
+                    <skip>${skipTests}</skip>
                     <connectors>
                         <connector implementation="org.mortbay.jetty.security.SslSocketConnector">
                             <port>41443</port>


Mime
View raw message