falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From peeyu...@apache.org
Subject falcon git commit: 89
Date Wed, 27 Apr 2016 16:02:46 GMT
Repository: falcon
Updated Branches:
  refs/heads/master 123391d46 -> 4ab28e80f


89

Pull request is available that will extend the cluster and process XSD to include the spark
execution engine.

Author: peeyush b <pbishnoi@hortonworks.com>

Reviewers: Pallavi <pallavi@inmobi.com>, Pavan <pavan.kolamuri@gmail.com>, Venkat
<n.r.v@live.com>, Venkatesan <me.venkatr@gmail.com>

Closes #89 from peeyushb/FALCON-1817 and squashes the following commits:

64d8c35 [peeyush b] making name as required parameter
b26af4e [peeyush b] removed line
8056916 [peeyush b] Incorporating review comments


Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/4ab28e80
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/4ab28e80
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/4ab28e80

Branch: refs/heads/master
Commit: 4ab28e80f908ad8f3aac65c2dd39eb7f49b213ae
Parents: 123391d
Author: Peeyush Bishnoi <peeyushb@apache.org>
Authored: Wed Apr 27 21:32:27 2016 +0530
Committer: peeyush b <pbishnoi@hortonworks.com>
Committed: Wed Apr 27 21:32:27 2016 +0530

----------------------------------------------------------------------
 client/src/main/resources/cluster-0.1.xsd |  2 ++
 client/src/main/resources/process-0.1.xsd | 32 ++++++++++++++++++++++++++
 2 files changed, 34 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/falcon/blob/4ab28e80/client/src/main/resources/cluster-0.1.xsd
----------------------------------------------------------------------
diff --git a/client/src/main/resources/cluster-0.1.xsd b/client/src/main/resources/cluster-0.1.xsd
index 34e3689..0e0ada8 100644
--- a/client/src/main/resources/cluster-0.1.xsd
+++ b/client/src/main/resources/cluster-0.1.xsd
@@ -167,6 +167,7 @@
                 messaging specifies the interface for sending feed availability messages,
it's
                 endpoint is broker url with tcp address.ex: tcp://corp.messaging:61616?daemon=true
                 registry specifies the interface for Hcatalog.
+                spark specifies the interface for spark master.
             </xs:documentation>
         </xs:annotation>
         <xs:restriction base="xs:string">
@@ -176,6 +177,7 @@
             <xs:enumeration value="workflow"/>
             <xs:enumeration value="messaging"/>
             <xs:enumeration value="registry"/>
+            <xs:enumeration value="spark"/>
         </xs:restriction>
     </xs:simpleType>
     <xs:simpleType name="IDENTIFIER">

http://git-wip-us.apache.org/repos/asf/falcon/blob/4ab28e80/client/src/main/resources/process-0.1.xsd
----------------------------------------------------------------------
diff --git a/client/src/main/resources/process-0.1.xsd b/client/src/main/resources/process-0.1.xsd
index 9d7898f..456ebf9 100644
--- a/client/src/main/resources/process-0.1.xsd
+++ b/client/src/main/resources/process-0.1.xsd
@@ -152,6 +152,13 @@
                     </xs:documentation>
                 </xs:annotation>
             </xs:element>
+            <xs:element type="spark-attributes" name="spark-attributes" minOccurs="0">
+                <xs:annotation>
+                    <xs:documentation>Defines the spark related attributes to run the
Spark application. Attributes
+                        must be defined when workflow engine "spark" is selected.
+                    </xs:documentation>
+                </xs:annotation>
+            </xs:element>
             <xs:element type="retry" name="retry" minOccurs="0">
                 <xs:annotation>
                     <xs:documentation>Retry defines how to handle workflow failures.
Policy type - periodic, exponential
@@ -331,9 +338,34 @@
             <xs:enumeration value="oozie"/>
             <xs:enumeration value="pig"/>
             <xs:enumeration value="hive"/>
+            <xs:enumeration value="spark"/>
         </xs:restriction>
     </xs:simpleType>
 
+    <xs:complexType name="spark-attributes">
+        <xs:annotation>
+            <xs:documentation>
+                List of Spark attributes to run Spark application:
+                    master element is optional and indicates the url of the Spark Master.
Ex: yarn-client, yarn-master, or local. This will override the Spark interface, if define
in cluster entity.
+                    mode element if present indicates the mode of spark, where to run spark
driver program. Ex: client,cluster.
+                    name element indicates the name of the spark application.
+                    class element if present, indicates the spark's application main class.
+                    jar element indicates a comma separated list of jars or python files.
Also provide the jar or file directory path to workflow attribute path
+                    spark-opts element if present, contains a list of spark options that
can be passed to spark driver.
+                    arg element if present, contains arguments that can be passed to spark
application.
+            </xs:documentation>
+        </xs:annotation>
+        <xs:sequence>
+            <xs:element name="master" type="xs:string" minOccurs="0" maxOccurs="1"/>
+            <xs:element name="mode" type="xs:string" minOccurs="0" maxOccurs="1"/>
+            <xs:element name="name" type="xs:string" minOccurs="1" maxOccurs="1"/>
+            <xs:element name="class" type="xs:string" minOccurs="0" maxOccurs="1"/>
+            <xs:element name="jar" type="xs:string" minOccurs="1" maxOccurs="1"/>
+            <xs:element name="spark-opts" type="xs:string" minOccurs="0" maxOccurs="1"/>
+            <xs:element name="arg" type="xs:string" minOccurs="0" maxOccurs="unbounded"/>
+        </xs:sequence>
+    </xs:complexType>
+
     <xs:complexType name="retry">
         <xs:attribute type="policy-type" name="policy" use="required"/>
         <xs:attribute type="frequency-type" name="delay" use="required"/>


Mime
View raw message