falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From prag...@apache.org
Subject falcon git commit: FALCON-1664 Add Postgres support for native scheduler.Contributed by Deepak Barr.
Date Wed, 27 Jan 2016 10:40:20 GMT
Repository: falcon
Updated Branches:
  refs/heads/master 093ad2797 -> 5ff837819


FALCON-1664 Add Postgres support for native scheduler.Contributed by Deepak Barr.


Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/5ff83781
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/5ff83781
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/5ff83781

Branch: refs/heads/master
Commit: 5ff837819f5a6f31070d5f8bfbecacdd1eeb0ab9
Parents: 093ad27
Author: Pragya <mittal.pragya23@gmail.com>
Authored: Wed Jan 27 10:31:35 2016 +0000
Committer: Pragya <mittal.pragya23@gmail.com>
Committed: Wed Jan 27 10:31:35 2016 +0000

----------------------------------------------------------------------
 CHANGES.txt                                     |  3 +++
 common/src/main/resources/statestore.properties |  2 +-
 docs/src/site/twiki/FalconNativeScheduler.twiki |  2 +-
 pom.xml                                         |  1 +
 scheduler/pom.xml                               | 10 ++++++++
 .../falcon/state/store/jdbc/InstanceBean.java   |  4 +--
 .../falcon/tools/FalconStateStoreDBCLI.java     |  3 ++-
 .../src/main/resources/META-INF/persistence.xml | 27 ++++++++++++++++++++
 src/conf/statestore.properties                  |  2 +-
 9 files changed, 48 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/falcon/blob/5ff83781/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index d87ce26..90b32ca 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -3,6 +3,9 @@ Trunk
   TASKS:
   INCOMPATIBLE CHANGES
   NEW FEATURES
+   
+    FALCON-1664 Add Postgres support for native scheduler(Deepak Barr via Pragya Mittal)
+
     FALCON-1495 In instance status list, show all runs for instances when requested by user(Narayan
Periwal via Ajay Yadava)
 
     FALCON-1230 Data based notification Service to notify execution instances when data becomes
available(Pavan Kumar Kolamuri via Ajay Yadava)

http://git-wip-us.apache.org/repos/asf/falcon/blob/5ff83781/common/src/main/resources/statestore.properties
----------------------------------------------------------------------
diff --git a/common/src/main/resources/statestore.properties b/common/src/main/resources/statestore.properties
index a67a871..44e79b3 100644
--- a/common/src/main/resources/statestore.properties
+++ b/common/src/main/resources/statestore.properties
@@ -21,7 +21,7 @@
 ######### StateStore Properties #####
 #*.falcon.state.store.impl=org.apache.falcon.state.store.jdbc.JDBCStateStore
 #*.falcon.statestore.jdbc.driver=org.apache.derby.jdbc.EmbeddedDriver
-## Falcon currently supports derby and mysql, change url based on DB.
+## Falcon currently supports derby, mysql and postgreSQL, change url based on DB.
 #*.falcon.statestore.jdbc.url=jdbc:derby:data/falcon.db;create=true
 
 ## StateStore credentials file where username,password and other properties can be stored
securely.

http://git-wip-us.apache.org/repos/asf/falcon/blob/5ff83781/docs/src/site/twiki/FalconNativeScheduler.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/FalconNativeScheduler.twiki b/docs/src/site/twiki/FalconNativeScheduler.twiki
index 6f0679e..de6bfe7 100644
--- a/docs/src/site/twiki/FalconNativeScheduler.twiki
+++ b/docs/src/site/twiki/FalconNativeScheduler.twiki
@@ -113,7 +113,7 @@ usage:
                                         schema
 
 </verbatim>
-Currently, MySQL and Derby are supported as state stores. We may extend support to other
DBs in the future. Falcon has been tested against MySQL v5.5.
+Currently, MySQL, postgreSQL and Derby are supported as state stores. We may extend support
to other DBs in the future. Falcon has been tested against MySQL v5.5 and PostgreSQL v9.5.
 
 ---++++ Using Derby as the State Store
 Using Derby is ideal for QA and staging setup. Falcon comes bundled with a Derby connector
and no explicit setup is required (although you can set it up) in terms creating the DB or
tables.

http://git-wip-us.apache.org/repos/asf/falcon/blob/5ff83781/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index a80822f..94f7aee 100644
--- a/pom.xml
+++ b/pom.xml
@@ -110,6 +110,7 @@
         <derby.version>10.10.1.1</derby.version>
         <commons-dbcp.version>1.4</commons-dbcp.version>
         <mysql.version>5.1.36</mysql.version>
+        <postgres.version>9.1-901-1.jdbc4</postgres.version>
         <internal.maven.repo>file:///tmp/falcontemprepo</internal.maven.repo>
         <excluded.test.groups>exhaustive</excluded.test.groups>
     </properties>

http://git-wip-us.apache.org/repos/asf/falcon/blob/5ff83781/scheduler/pom.xml
----------------------------------------------------------------------
diff --git a/scheduler/pom.xml b/scheduler/pom.xml
index 6392e98..2e2b2e9 100644
--- a/scheduler/pom.xml
+++ b/scheduler/pom.xml
@@ -145,6 +145,12 @@
         </dependency>
 
         <dependency>
+            <groupId>postgresql</groupId>
+            <artifactId>postgresql</artifactId>
+            <version>${postgres.version}</version>
+        </dependency>
+
+        <dependency>
             <groupId>commons-dbcp</groupId>
             <artifactId>commons-dbcp</artifactId>
             <version>${commons-dbcp.version}</version>
@@ -207,6 +213,10 @@
                                 <groupId>mysql</groupId>
                                 <artifactId>mysql-connector-java</artifactId>
                             </artifactItem>
+                            <artifactItem>
+                                <groupId>postgresql</groupId>
+                                <artifactId>postgresql</artifactId>
+                            </artifactItem>
                         </artifactItems>
                         <outputDirectory>${project.build.directory}/dependency</outputDirectory>
                     </configuration>

http://git-wip-us.apache.org/repos/asf/falcon/blob/5ff83781/scheduler/src/main/java/org/apache/falcon/state/store/jdbc/InstanceBean.java
----------------------------------------------------------------------
diff --git a/scheduler/src/main/java/org/apache/falcon/state/store/jdbc/InstanceBean.java
b/scheduler/src/main/java/org/apache/falcon/state/store/jdbc/InstanceBean.java
index e8385b1..dffb116 100644
--- a/scheduler/src/main/java/org/apache/falcon/state/store/jdbc/InstanceBean.java
+++ b/scheduler/src/main/java/org/apache/falcon/state/store/jdbc/InstanceBean.java
@@ -114,11 +114,11 @@ public class InstanceBean {
     private EntityBean entityBean;
 
 
-    @Column(name = "awaited_predicates", columnDefinition = "BLOB")
+    @Column(name = "awaited_predicates")
     @Lob
     private byte[] awaitedPredicates;
 
-    @Column(name = "properties", columnDefinition = "BLOB")
+    @Column(name = "properties")
     @Lob
     private byte[] properties;
 

http://git-wip-us.apache.org/repos/asf/falcon/blob/5ff83781/scheduler/src/main/java/org/apache/falcon/tools/FalconStateStoreDBCLI.java
----------------------------------------------------------------------
diff --git a/scheduler/src/main/java/org/apache/falcon/tools/FalconStateStoreDBCLI.java b/scheduler/src/main/java/org/apache/falcon/tools/FalconStateStoreDBCLI.java
index 7f22b0e..6de9f7d 100644
--- a/scheduler/src/main/java/org/apache/falcon/tools/FalconStateStoreDBCLI.java
+++ b/scheduler/src/main/java/org/apache/falcon/tools/FalconStateStoreDBCLI.java
@@ -51,7 +51,8 @@ public class FalconStateStoreDBCLI {
 
     // Represents whether DB instance exists or not.
     private boolean instanceExists;
-    private static final String[] FALCON_HELP = {"Falcon DB initialization tool currently
supports Derby DB/ Mysql"};
+    private static final String[] FALCON_HELP =
+    {"Falcon DB initialization tool currently supports Derby DB/ Mysql/ PostgreSQL"};
 
     public static void main(String[] args) {
         new FalconStateStoreDBCLI().run(args);

http://git-wip-us.apache.org/repos/asf/falcon/blob/5ff83781/scheduler/src/main/resources/META-INF/persistence.xml
----------------------------------------------------------------------
diff --git a/scheduler/src/main/resources/META-INF/persistence.xml b/scheduler/src/main/resources/META-INF/persistence.xml
index 99e116c..c2ef794 100644
--- a/scheduler/src/main/resources/META-INF/persistence.xml
+++ b/scheduler/src/main/resources/META-INF/persistence.xml
@@ -74,4 +74,31 @@
         </properties>
     </persistence-unit>
 
+    <persistence-unit name="falcon-postgresql" transaction-type="RESOURCE_LOCAL">
+        <provider>org.apache.openjpa.persistence.PersistenceProviderImpl</provider>
+
+        <class>org.apache.falcon.state.store.jdbc.EntityBean</class>
+        <class>org.apache.falcon.state.store.jdbc.InstanceBean</class>
+
+        <properties>
+            <property name="openjpa.ConnectionDriverName" value="org.apache.commons.dbcp.BasicDataSource"/>
+
+            <property name="openjpa.ConnectionProperties" value="**INVALID**"/> <!--Set
by StoreService at init time -->
+
+            <property name="openjpa.MetaDataFactory"
+                      value="jpa(Types=org.apache.falcon.state.store.EntityBean;
+                org.apache.falcon.state.store.InstanceBean)"></property>
+
+            <property name="openjpa.DetachState" value="fetch-groups(DetachedStateField=true)"/>
+            <property name="openjpa.LockManager" value="pessimistic"/>
+            <property name="openjpa.ReadLockLevel" value="read"/>
+            <property name="openjpa.WriteLockLevel" value="write"/>
+            <property name="openjpa.jdbc.TransactionIsolation" value="repeatable-read"/>
<!--CUSTOM-->
+            <property name="openjpa.jdbc.DBDictionary" value="batchLimit=50"/>
+            <property name="openjpa.jdbc.DBDictionary" value="TimestampTypeName=TIMESTAMP"/>
+            <property name="openjpa.RuntimeUnenhancedClasses" value="unsupported"/>
+            <property name="openjpa.Log" value="log4j"/>
+        </properties>
+    </persistence-unit>
+
 </persistence>

http://git-wip-us.apache.org/repos/asf/falcon/blob/5ff83781/src/conf/statestore.properties
----------------------------------------------------------------------
diff --git a/src/conf/statestore.properties b/src/conf/statestore.properties
index de5ad8f..5659531 100644
--- a/src/conf/statestore.properties
+++ b/src/conf/statestore.properties
@@ -21,7 +21,7 @@
 ######### StateStore Properties #####
 #*.falcon.state.store.impl=org.apache.falcon.state.store.jdbc.JDBCStateStore
 #*.falcon.statestore.jdbc.driver=org.apache.derby.jdbc.EmbeddedDriver
-## Falcon currently supports derby and mysql, change url based on DB.
+## Falcon currently supports derby, mysql and postgreSQL, change url based on DB.
 #*.falcon.statestore.jdbc.url=jdbc:derby:data/falcon.db;create=true
 
 ## StateStore credentials file where username,password and other properties can be stored
securely.


Mime
View raw message