falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ajayyad...@apache.org
Subject falcon git commit: FALCON-1658 MySql Support for Native Scheduler. Contributed by Pavan Kumar Kolamuri.
Date Mon, 14 Dec 2015 17:50:46 GMT
Repository: falcon
Updated Branches:
  refs/heads/master 3059980c6 -> 4445798be


FALCON-1658 MySql Support for Native Scheduler. Contributed by Pavan Kumar Kolamuri.


Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/4445798b
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/4445798b
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/4445798b

Branch: refs/heads/master
Commit: 4445798bebd0420d86b5f091354b1155a3a09f77
Parents: 3059980
Author: Ajay Yadava <ajaynsit@gmail.com>
Authored: Mon Dec 14 19:53:02 2015 +0530
Committer: Ajay Yadava <ajaynsit@gmail.com>
Committed: Mon Dec 14 19:55:04 2015 +0530

----------------------------------------------------------------------
 CHANGES.txt                                     |  2 ++
 pom.xml                                         |  1 +
 scheduler/pom.xml                               | 13 ++++++++--
 .../falcon/tools/FalconStateStoreDBCLI.java     |  2 +-
 .../src/main/resources/META-INF/persistence.xml | 27 ++++++++++++++++++++
 src/bin/falcon-db.sh                            |  2 +-
 src/conf/startup.properties                     | 25 ++++++++++++++----
 7 files changed, 63 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/falcon/blob/4445798b/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 4d42881..b594160 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -31,6 +31,8 @@ Trunk (Unreleased)
     FALCON-1213 Base framework of the native scheduler(Pallavi Rao)
 
   IMPROVEMENTS
+    FALCON-1658 MySql Support for Native Scheduler(Pavan Kumar Kolamuri via Ajay Yadava)
+
     FALCON-1656 Improve FeedHelper:getRetentionFrequency method(Ajay Yadava)
 
     FALCON-1616 Consume Workflow job end notifications for SLA monitoring(Sandeep Samudrala
via Ajay Yadava)

http://git-wip-us.apache.org/repos/asf/falcon/blob/4445798b/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 3ad8789..a7b21ae 100644
--- a/pom.xml
+++ b/pom.xml
@@ -117,6 +117,7 @@
         <javax-validation.version>1.0.0.GA</javax-validation.version>
         <derby.version>10.10.1.1</derby.version>
         <commons-dbcp.version>1.4</commons-dbcp.version>
+        <mysql.version>5.1.36</mysql.version>
         <internal.maven.repo>file:///tmp/falcontemprepo</internal.maven.repo>
         <excluded.test.groups>exhaustive</excluded.test.groups>
     </properties>

http://git-wip-us.apache.org/repos/asf/falcon/blob/4445798b/scheduler/pom.xml
----------------------------------------------------------------------
diff --git a/scheduler/pom.xml b/scheduler/pom.xml
index c305651..c934b3e 100644
--- a/scheduler/pom.xml
+++ b/scheduler/pom.xml
@@ -135,7 +135,13 @@
         <dependency>
             <groupId>org.apache.derby</groupId>
             <artifactId>derby</artifactId>
-            <version>10.10.1.1</version>
+            <version>${derby.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>mysql</groupId>
+            <artifactId>mysql-connector-java</artifactId>
+            <version>${mysql.version}</version>
         </dependency>
 
         <dependency>
@@ -196,7 +202,10 @@
                             <artifactItem>
                                 <groupId>commons-dbcp</groupId>
                                 <artifactId>commons-dbcp</artifactId>
-                                <version>${commons-dbcp.version}</version>
+                            </artifactItem>
+                            <artifactItem>
+                                <groupId>mysql</groupId>
+                                <artifactId>mysql-connector-java</artifactId>
                             </artifactItem>
                         </artifactItems>
                         <outputDirectory>${project.build.directory}/dependency</outputDirectory>

http://git-wip-us.apache.org/repos/asf/falcon/blob/4445798b/scheduler/src/main/java/org/apache/falcon/tools/FalconStateStoreDBCLI.java
----------------------------------------------------------------------
diff --git a/scheduler/src/main/java/org/apache/falcon/tools/FalconStateStoreDBCLI.java b/scheduler/src/main/java/org/apache/falcon/tools/FalconStateStoreDBCLI.java
index 381b0b3..f4058c8 100644
--- a/scheduler/src/main/java/org/apache/falcon/tools/FalconStateStoreDBCLI.java
+++ b/scheduler/src/main/java/org/apache/falcon/tools/FalconStateStoreDBCLI.java
@@ -51,7 +51,7 @@ public class FalconStateStoreDBCLI {
 
     // Represents whether DB instance exists or not.
     private boolean instanceExists;
-    private static final String[] FALCON_HELP = {"Falcon DB initialization tool currently
supports Derby DB"};
+    private static final String[] FALCON_HELP = {"Falcon DB initialization tool currently
supports Derby DB/ Mysql"};
 
     public static void main(String[] args) {
         new FalconStateStoreDBCLI().run(args);

http://git-wip-us.apache.org/repos/asf/falcon/blob/4445798b/scheduler/src/main/resources/META-INF/persistence.xml
----------------------------------------------------------------------
diff --git a/scheduler/src/main/resources/META-INF/persistence.xml b/scheduler/src/main/resources/META-INF/persistence.xml
index 86558de..99e116c 100644
--- a/scheduler/src/main/resources/META-INF/persistence.xml
+++ b/scheduler/src/main/resources/META-INF/persistence.xml
@@ -47,4 +47,31 @@
         </properties>
     </persistence-unit>
 
+    <persistence-unit name="falcon-mysql" transaction-type="RESOURCE_LOCAL">
+        <provider>org.apache.openjpa.persistence.PersistenceProviderImpl</provider>
+
+        <class>org.apache.falcon.state.store.jdbc.EntityBean</class>
+        <class>org.apache.falcon.state.store.jdbc.InstanceBean</class>
+
+        <properties>
+            <property name="openjpa.ConnectionDriverName" value="org.apache.commons.dbcp.BasicDataSource"/>
+
+            <property name="openjpa.ConnectionProperties" value="**INVALID**"/> <!--Set
by StoreService at init time -->
+
+            <property name="openjpa.MetaDataFactory"
+                      value="jpa(Types=org.apache.falcon.state.store.EntityBean;
+                org.apache.falcon.state.store.InstanceBean)"></property>
+
+            <property name="openjpa.DetachState" value="fetch-groups(DetachedStateField=true)"/>
+            <property name="openjpa.LockManager" value="pessimistic"/>
+            <property name="openjpa.ReadLockLevel" value="read"/>
+            <property name="openjpa.WriteLockLevel" value="write"/>
+            <property name="openjpa.jdbc.TransactionIsolation" value="repeatable-read"/>
<!--CUSTOM-->
+            <property name="openjpa.jdbc.DBDictionary" value="batchLimit=50"/>
+            <property name="openjpa.jdbc.DBDictionary" value="TimestampTypeName=TIMESTAMP"/>
+            <property name="openjpa.RuntimeUnenhancedClasses" value="unsupported"/>
+            <property name="openjpa.Log" value="log4j"/>
+        </properties>
+    </persistence-unit>
+
 </persistence>

http://git-wip-us.apache.org/repos/asf/falcon/blob/4445798b/src/bin/falcon-db.sh
----------------------------------------------------------------------
diff --git a/src/bin/falcon-db.sh b/src/bin/falcon-db.sh
index 415fd5d..bdeaf8c 100644
--- a/src/bin/falcon-db.sh
+++ b/src/bin/falcon-db.sh
@@ -46,4 +46,4 @@ done
 # set the client class path
 FALCONCPPATH="$FALCONCPPATH:${BASEDIR}/client/lib/*"
 
-${JAVA_BIN} ${JAVA_PROPERTIES} -cp ${FALCONCPPATH} org.apache.falcon.tools.FalconStateStoreDBCLI
"${@}"
+${JAVA_BIN} ${JAVA_PROPERTIES} -cp ${FALCONCPPATH} -Dfalcon.log.dir=$HOME org.apache.falcon.tools.FalconStateStoreDBCLI
"${@}"

http://git-wip-us.apache.org/repos/asf/falcon/blob/4445798b/src/conf/startup.properties
----------------------------------------------------------------------
diff --git a/src/conf/startup.properties b/src/conf/startup.properties
index 1694930..95d792b 100644
--- a/src/conf/startup.properties
+++ b/src/conf/startup.properties
@@ -27,7 +27,6 @@
 
 ## If you wish to use Falcon native scheduler uncomment and enable the properties below.##
 #*.dag.engine.impl=org.apache.falcon.workflow.engine.OozieDAGEngine
-#*.state.store.impl=org.apache.falcon.state.store.jdbc.JdbcStateStore
 ## If you wish to use Falcon native scheduler set the workflow engine to FalconWorkflowEngine
instead of OozieWorkflowEngine. ##
 #*.workflow.engine.impl=org.apache.falcon.workflow.engine.FalconWorkflowEngine
 *.workflow.engine.impl=org.apache.falcon.workflow.engine.OozieWorkflowEngine
@@ -52,13 +51,28 @@
                         org.apache.falcon.service.LogCleanupService,\
                         org.apache.falcon.service.GroupsService,\
                         org.apache.falcon.service.ProxyUserService
-## If you wish to use Falcon native scheduler add the commented out services below to application.services
##
+## If you wish to use Falcon native scheduler uncomment out below  application services and
+# comment out above application
+#
+#services ##
+#*.application.services=org.apache.falcon.security.AuthenticationInitializationService,\
+#                        org.apache.falcon.workflow.WorkflowJobEndNotificationService, \
+#                        org.apache.falcon.service.ProcessSubscriberService,\
+#                        org.apache.falcon.service.FeedSLAMonitoringService,\
+#                        org.apache.falcon.service.LifecyclePolicyMap,\
+#                        org.apache.falcon.state.store.service.FalconJPAService,\
+#                        org.apache.falcon.entity.store.ConfigurationStore,\
+#                        org.apache.falcon.rerun.service.RetryService,\
+#                        org.apache.falcon.rerun.service.LateRunService,\
+#                        org.apache.falcon.metadata.MetadataMappingService,\
+#                        org.apache.falcon.service.LogCleanupService,\
+#                        org.apache.falcon.service.GroupsService,\
+#                        org.apache.falcon.service.ProxyUserService,\
 #                        org.apache.falcon.notification.service.impl.JobCompletionService,\
 #                        org.apache.falcon.notification.service.impl.SchedulerService,\
 #                        org.apache.falcon.notification.service.impl.AlarmService,\
 #                        org.apache.falcon.notification.service.impl.DataAvailabilityService,\
-#                        org.apache.falcon.execution.FalconExecutionService,\
-#                        org.apache.falcon.state.store.service.FalconJPAService
+#                        org.apache.falcon.execution.FalconExecutionService
 
 ##### Prism Services #####
 prism.application.services=org.apache.falcon.service.LifecyclePolicyMap,\
@@ -78,7 +92,7 @@ prism.application.services=org.apache.falcon.service.LifecyclePolicyMap,\
                         org.apache.falcon.service.FeedSLAMonitoringService,\
                         org.apache.falcon.service.SharedLibraryHostingService
 ## If you wish to use Falcon native scheduler, add the State store as a configstore listener.
##
-#                       org.apache.falcon.state.store.jdbc.JdbcStateStore
+#                       org.apache.falcon.state.store.jdbc.JDBCStateStore
 
 ##### Prism Configuration Store Change listeners #####
 prism.configstore.listeners=org.apache.falcon.entity.v0.EntityGraph,\
@@ -267,6 +281,7 @@ prism.configstore.listeners=org.apache.falcon.entity.v0.EntityGraph,\
 ######### StateStore Properties #####
 #*.falcon.state.store.impl=org.apache.falcon.state.store.jdbc.JDBCStateStore
 #*.falcon.statestore.jdbc.driver=org.apache.derby.jdbc.EmbeddedDriver
+## Falcon currently supports derby and mysql, change url based on DB.
 #*.falcon.statestore.jdbc.url=jdbc:derby:data/statestore.db;create=true
 #*.falcon.statestore.jdbc.username=sa
 #*.falcon.statestore.jdbc.password=


Mime
View raw message