falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From b...@apache.org
Subject falcon git commit: FALCON-1984 Provide proper hint and documentation if required titan storage backend is not configured in startup.properties
Date Fri, 03 Jun 2016 14:53:19 GMT
Repository: falcon
Updated Branches:
  refs/heads/master 16d2b39b2 -> e84fca7ff


FALCON-1984 Provide proper hint and documentation if required titan storage backend is not
configured in startup.properties

Added validation of required configuration properties for graph db storage backend.
Added error hints on the missing property in Falcon server log.
Reorganized the order of the configuration properties in startup.properties so relevant properties
for each backend are grouped in consecutive lines.

Author: yzheng-hortonworks <yzheng@hortonworks.com>

Reviewers: "Balu Vellanki <balu@apache.org>, Venkat Ranganathan <venkat@hortonworks.com>"

Closes #166 from yzheng-hortonworks/FALCON-1984


Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/e84fca7f
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/e84fca7f
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/e84fca7f

Branch: refs/heads/master
Commit: e84fca7ff2ce235a7d804b3c4806091a2a2c3037
Parents: 16d2b39
Author: yzheng-hortonworks <yzheng@hortonworks.com>
Authored: Fri Jun 3 07:53:16 2016 -0700
Committer: bvellanki <bvellanki@hortonworks.com>
Committed: Fri Jun 3 07:53:16 2016 -0700

----------------------------------------------------------------------
 .../falcon/metadata/MetadataMappingService.java | 72 +++++++++++++++++++-
 common/src/main/resources/startup.properties    | 17 ++++-
 .../metadata/MetadataMappingServiceTest.java    |  6 +-
 docs/src/site/twiki/Configuration.twiki         | 10 ++-
 .../resource/metadata/MetadataTestContext.java  |  4 ++
 src/conf/startup.properties                     | 18 ++---
 6 files changed, 106 insertions(+), 21 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/falcon/blob/e84fca7f/common/src/main/java/org/apache/falcon/metadata/MetadataMappingService.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/metadata/MetadataMappingService.java b/common/src/main/java/org/apache/falcon/metadata/MetadataMappingService.java
index 66a3a58..727be56 100644
--- a/common/src/main/java/org/apache/falcon/metadata/MetadataMappingService.java
+++ b/common/src/main/java/org/apache/falcon/metadata/MetadataMappingService.java
@@ -35,6 +35,7 @@ import com.tinkerpop.blueprints.util.TransactionWork;
 import org.apache.commons.configuration.BaseConfiguration;
 import org.apache.commons.configuration.Configuration;
 import org.apache.falcon.FalconException;
+import org.apache.falcon.FalconRuntimException;
 import org.apache.falcon.entity.store.ConfigurationStore;
 import org.apache.falcon.entity.v0.Entity;
 import org.apache.falcon.entity.v0.EntityType;
@@ -48,6 +49,9 @@ import org.slf4j.LoggerFactory;
 import org.apache.falcon.workflow.WorkflowExecutionContext;
 import org.apache.falcon.workflow.WorkflowExecutionListener;
 
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
 import java.util.Map;
 import java.util.Properties;
 import java.util.Set;
@@ -69,7 +73,26 @@ public class MetadataMappingService
      * Constant for the configuration property that indicates the prefix.
      */
     private static final String FALCON_PREFIX = "falcon.graph.";
-
+    /**
+     * Constant for the configuration property that indicates the storage backend.
+     */
+    public static final String PROPERTY_KEY_STORAGE_BACKEND = "storage.backend";
+    public static final String STORAGE_BACKEND_HBASE = "hbase";
+    public static final String STORAGE_BACKEND_BDB = "berkeleyje";
+    /**
+     * HBase configuration properties.
+     */
+    public static final String PROPERTY_KEY_STORAGE_HOSTNAME = "storage.hostname";
+    public static final String PROPERTY_KEY_STORAGE_TABLE = "storage.hbase.table";
+    public static final Set<String> PROPERTY_KEYS_HBASE = Collections.unmodifiableSet(new
HashSet<>(Arrays.asList(
+            PROPERTY_KEY_STORAGE_HOSTNAME, PROPERTY_KEY_STORAGE_TABLE)));
+    /**
+     * Berkeley DB configuration properties.
+     */
+    public static final String PROPERTY_KEY_STORAGE_DIRECTORY = "storage.directory";
+    public static final String PROPERTY_KEY_SERIALIZE_PATH = "serialize.path";
+    public static final Set<String> PROPERTY_KEYS_BDB = Collections.unmodifiableSet(new
HashSet<>(Arrays.asList(
+            PROPERTY_KEY_STORAGE_DIRECTORY, PROPERTY_KEY_SERIALIZE_PATH)));
 
     private Graph graph;
     private Set<String> vertexIndexedKeys;
@@ -118,11 +141,56 @@ public class MetadataMappingService
 
     protected Graph initializeGraphDB() {
         LOG.info("Initializing graph db");
-
         Configuration graphConfig = getConfiguration();
+        validateConfiguration(graphConfig);
         return GraphFactory.open(graphConfig);
     }
 
+    private void validateConfiguration(Configuration graphConfig) {
+        // check if storage backend if configured
+        if (!graphConfig.containsKey(PROPERTY_KEY_STORAGE_BACKEND)) {
+            throw new FalconRuntimException("Titan GraphDB storage backend is not configured.
"
+                    + "You need to choose either hbase or berkeleydb."
+                    + "Please check Configuration twiki or "
+                    + "the section Graph Database Properties in startup.properties "
+                    + "on how to configure Titan GraphDB backend.");
+        }
+
+        String backend = graphConfig.getString(PROPERTY_KEY_STORAGE_BACKEND);
+        switch (backend) {
+        case STORAGE_BACKEND_BDB:
+            // check required parameter for Berkeley DB backend
+            for (String key : PROPERTY_KEYS_BDB) {
+                if (!graphConfig.containsKey(key)) {
+                    throw new FalconRuntimException("Required parameter " + FALCON_PREFIX
+ key
+                            + " not found in startup.properties."
+                            + "Please check Configuration twiki or "
+                            + "the section Graph Database Properties in startup.properties
"
+                            + "on how to configure Berkeley DB storage backend.");
+                }
+            }
+            break;
+        case STORAGE_BACKEND_HBASE:
+            // check required parameter for HBase backend
+            for (String key : PROPERTY_KEYS_HBASE) {
+                if (!graphConfig.containsKey(key)) {
+                    throw new FalconRuntimException("Required parameter " + FALCON_PREFIX
+ key
+                            + " not found in startup.properties."
+                            + "Please check Configuration twiki or "
+                            + "the section Graph Database Properties in startup.properties
"
+                            + "on how to configure HBase storage backend.");
+                }
+            }
+            break;
+        default:
+            throw new FalconRuntimException("Invalid graph storage backend: " + backend +
". "
+                    + "You need to choose either hbase or berkeleydb."
+                    + "Please check Configuration twiki or "
+                    + "the section Graph Database Properties in startup.properties "
+                    + "on how to configure Titan GraphDB backend.");
+        }
+    }
+
     public static Configuration getConfiguration() {
         Configuration graphConfig = new BaseConfiguration();
 

http://git-wip-us.apache.org/repos/asf/falcon/blob/e84fca7f/common/src/main/resources/startup.properties
----------------------------------------------------------------------
diff --git a/common/src/main/resources/startup.properties b/common/src/main/resources/startup.properties
index 2229edf..8ce1174 100644
--- a/common/src/main/resources/startup.properties
+++ b/common/src/main/resources/startup.properties
@@ -156,9 +156,20 @@ it.workflow.execution.listeners=org.apache.falcon.catalog.CatalogPartitionHandle
 *.falcon.graph.blueprints.graph=com.thinkaurelius.titan.core.TitanFactory
 
 # Graph Storage
-*.falcon.graph.storage.directory=${user.dir}/target/graphdb
-*.falcon.graph.storage.backend=berkeleyje
-*.falcon.graph.serialize.path=${user.dir}/target/graphdb
+# IMPORTANT:   Please enable one of the graph db backend: hbase or berkeleydb, per instructions
below.
+
+# Enable the following for Berkeley DB.  Make sure je-5.0.73.jar is downloaded and available
+# under Falcon webapp directory or under falcon server classpath.
+#*.falcon.graph.storage.backend=berkeleyje
+#*.falcon.graph.storage.directory=/${falcon.home}/data/graphdb
+#*.falcon.graph.serialize.path=${user.dir}/target/graphdb
+
+# Enable the following for HBase
+#*.falcon.graph.storage.backend=hbase
+# For standalone mode , set hostname to localhost; for distributed mode, set to the zookeeper
quorum
+# @see http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2
+#*.falcon.graph.storage.hostname=localhost
+#*.falcon.graph.storage.hbase.table=falcon_titan
 
 # Avoid acquiring read lock when iterating over large graphs
 # See http://s3.thinkaurelius.com/docs/titan/0.5.4/bdb.html

http://git-wip-us.apache.org/repos/asf/falcon/blob/e84fca7f/common/src/test/java/org/apache/falcon/metadata/MetadataMappingServiceTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/falcon/metadata/MetadataMappingServiceTest.java
b/common/src/test/java/org/apache/falcon/metadata/MetadataMappingServiceTest.java
index c0ae5fc..62db501 100644
--- a/common/src/test/java/org/apache/falcon/metadata/MetadataMappingServiceTest.java
+++ b/common/src/test/java/org/apache/falcon/metadata/MetadataMappingServiceTest.java
@@ -122,8 +122,10 @@ public class MetadataMappingServiceTest {
         configStore = ConfigurationStore.get();
 
         Services.get().register(new WorkflowJobEndNotificationService());
-        StartupProperties.get().setProperty("falcon.graph.storage.directory",
-                "target/graphdb-" + System.currentTimeMillis());
+        StartupProperties.get().setProperty("falcon.graph.storage.backend", "berkeleyje");
+        String graphDBDir = "target/graphdb-" + System.currentTimeMillis();
+        StartupProperties.get().setProperty("falcon.graph.storage.directory", graphDBDir);
+        StartupProperties.get().setProperty("falcon.graph.serialize.path", graphDBDir);
         StartupProperties.get().setProperty("falcon.graph.preserve.history", "true");
         service = new MetadataMappingService();
         service.init();

http://git-wip-us.apache.org/repos/asf/falcon/blob/e84fca7f/docs/src/site/twiki/Configuration.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/Configuration.twiki b/docs/src/site/twiki/Configuration.twiki
index bfca3d8..33eba20 100644
--- a/docs/src/site/twiki/Configuration.twiki
+++ b/docs/src/site/twiki/Configuration.twiki
@@ -325,10 +325,16 @@ to <verbatim>$FALCON_HOME/conf/startup.properties</verbatim>
before starting the
 For details on the same, refer to [[FalconNativeScheduler][Falcon Native Scheduler]]
 
 ---+++Titan GraphDB backend
-You can either choose to use 5.0.73 version of berkeleydb (the default for Falcon for the
last few releases) or 1.1.x or later version HBase as the backend database. Falcon in its
release distributions will have the titan storage plugins for both BerkeleyDB and HBase.
+GraphDB backend needs to be configured to properly start Falcon server.
+You can either choose to use 5.0.73 version of berkeleydb (the default for Falcon for the
last few releases) or 1.1.x or later version HBase as the backend database.
+Falcon in its release distributions will have the titan storage plugins for both BerkeleyDB
and HBase.
 
 ----++++Using BerkeleyDB backend
-Falcon distributions may not package berkeley db artifacts (je-5.0.73.jar) based on build
profiles.  If Berkeley DB is not packaged, you can download the Berkeley DB jar file from
the URL: <verbatim>http://download.oracle.com/otn/berkeley-db/je-5.0.73.zip</verbatim>.
  The following properties describe an example berkeley db graph storage backend that can
be specified in the configuration file <verbatim>$FALCON_HOME/conf/startup.properties</verbatim>.
+Falcon distributions may not package berkeley db artifacts (je-5.0.73.jar) based on build
profiles.
+If Berkeley DB is not packaged, you can download the Berkeley DB jar file from the URL:
+<verbatim>http://download.oracle.com/otn/berkeley-db/je-5.0.73.zip</verbatim>.
+The following properties describe an example berkeley db graph storage backend that can be
specified in the configuration file
+<verbatim>$FALCON_HOME/conf/startup.properties</verbatim>.
 
 <verbatim>
 # Graph Storage

http://git-wip-us.apache.org/repos/asf/falcon/blob/e84fca7f/prism/src/test/java/org/apache/falcon/resource/metadata/MetadataTestContext.java
----------------------------------------------------------------------
diff --git a/prism/src/test/java/org/apache/falcon/resource/metadata/MetadataTestContext.java
b/prism/src/test/java/org/apache/falcon/resource/metadata/MetadataTestContext.java
index 47d6ba1..0fc708d 100644
--- a/prism/src/test/java/org/apache/falcon/resource/metadata/MetadataTestContext.java
+++ b/prism/src/test/java/org/apache/falcon/resource/metadata/MetadataTestContext.java
@@ -95,6 +95,10 @@ public class MetadataTestContext {
         Services.get().register(new WorkflowJobEndNotificationService());
         Assert.assertTrue(Services.get().isRegistered(WorkflowJobEndNotificationService.SERVICE_NAME));
 
+        StartupProperties.get().setProperty("falcon.graph.storage.backend", "berkeleyje");
+        String graphDBDir = "target/graphdb-" + System.currentTimeMillis();
+        StartupProperties.get().setProperty("falcon.graph.storage.directory", graphDBDir);
+        StartupProperties.get().setProperty("falcon.graph.serialize.path", graphDBDir);
         StartupProperties.get().setProperty("falcon.graph.preserve.history", "true");
         service = new MetadataMappingService();
         service.init();

http://git-wip-us.apache.org/repos/asf/falcon/blob/e84fca7f/src/conf/startup.properties
----------------------------------------------------------------------
diff --git a/src/conf/startup.properties b/src/conf/startup.properties
index d732013..00ac4cd 100644
--- a/src/conf/startup.properties
+++ b/src/conf/startup.properties
@@ -173,25 +173,19 @@ prism.configstore.listeners=org.apache.falcon.entity.v0.EntityGraph,\
 *.falcon.graph.blueprints.graph=com.thinkaurelius.titan.core.TitanFactory
 
 # Graph Storage
-# IMPORTANT:   Please enable one of hbase or berkeleydb backends are enabled
-#  after the backend requirements are provisioned as needed.
+# IMPORTANT:   Please enable one of the graph db backend: hbase or berkeleydb, per instructions
below.
 
-# Enable the following for Berkeley DB.  Make sure je-5.0.73.jar is
-# downloaded and available under Falcon webapp directory or under falcon
-# server classpath.
-
-#*.falcon.graph.storage.directory=/${falcon.home}/data/graphdb
+# Enable the following for Berkeley DB.  Make sure je-5.0.73.jar is downloaded and available
+# under Falcon webapp directory or under falcon server classpath.
 #*.falcon.graph.storage.backend=berkeleyje
-
+#*.falcon.graph.storage.directory=/${falcon.home}/data/graphdb
+#*.falcon.graph.serialize.path=${user.dir}/target/graphdb
 
 # Enable the following for HBase
 #*.falcon.graph.storage.backend=hbase
-#For standalone mode , set hostname to localhost
-#for distributed mode, set to the zookeeper quorum
+# For standalone mode , set hostname to localhost; for distributed mode, set to the zookeeper
quorum
 # @see http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2
-
 #*.falcon.graph.storage.hostname=localhost
-#*.falcon.graph.serialize.path=${user.dir}/target/graphdb
 #*.falcon.graph.storage.hbase.table=falcon_titan
 
 # Avoid acquiring read lock when iterating over large graphs


Mime
View raw message