falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From b...@apache.org
Subject falcon git commit: FALCON-2104 Loss of data in GraphDB when upgrading Falcon 0.9 to 0.10
Date Tue, 02 Aug 2016 21:44:50 GMT
Repository: falcon
Updated Branches:
  refs/heads/0.10 eddf2ee5e -> 805298520


FALCON-2104 Loss of data in GraphDB when upgrading Falcon 0.9 to 0.10

Author: bvellanki <bvellanki@hortonworks.com>

Reviewers: "Venkat  Ranganathan <venkat@hortonworks.com>, Sowmya Ramesh <sowmya_kr@apache.org>"

Closes #253 from bvellanki/FALCON-2104

(cherry picked from commit dda42366823e0fdb996ed70e34710fa8dfb4393c)
Signed-off-by: bvellanki <bvellanki@hortonworks.com>


Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/80529852
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/80529852
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/80529852

Branch: refs/heads/0.10
Commit: 80529852009e8dfd3fd710c0c4ef062946136c3d
Parents: eddf2ee
Author: bvellanki <bvellanki@hortonworks.com>
Authored: Tue Aug 2 14:44:30 2016 -0700
Committer: bvellanki <bvellanki@hortonworks.com>
Committed: Tue Aug 2 14:44:44 2016 -0700

----------------------------------------------------------------------
 .../falcon/metadata/GraphUpdateUtils.java       | 113 ++++++++++++++++++
 .../falcon/metadata/MetadataMappingService.java |   4 +-
 docs/src/site/twiki/InstallationSteps.twiki     |  15 ++-
 docs/src/site/twiki/MigrationInstructions.twiki |  29 ++++-
 release-docs/0.10/CHANGES.0.10.md               |   1 +
 src/bin/graphdbutil.sh                          | 118 +++++++++++++++++++
 6 files changed, 264 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/falcon/blob/80529852/common/src/main/java/org/apache/falcon/metadata/GraphUpdateUtils.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/metadata/GraphUpdateUtils.java b/common/src/main/java/org/apache/falcon/metadata/GraphUpdateUtils.java
new file mode 100644
index 0000000..a256e46
--- /dev/null
+++ b/common/src/main/java/org/apache/falcon/metadata/GraphUpdateUtils.java
@@ -0,0 +1,113 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.falcon.metadata;
+
+import com.tinkerpop.blueprints.Graph;
+import com.tinkerpop.blueprints.util.io.graphson.GraphSONReader;
+import com.tinkerpop.blueprints.util.io.graphson.GraphSONWriter;
+import org.apache.commons.configuration.Configuration;
+import org.apache.commons.io.FileUtils;
+import org.apache.falcon.FalconException;
+
+import java.io.File;
+
+/**
+ * Utility class for graph operations.
+ */
+public final class GraphUpdateUtils {
+
+    private static final String BANNER_MSG =
+            "Before running this utility please make sure that Falcon startup properties
"
+                    + "has the right configuration settings for the graph database, "
+                    + "Falcon server is stopped and no other access to the graph database
is being performed.";
+
+    private static final String IMPORT = "import";
+    private static final String EXPORT = "export";
+    private static final String INSTANCE_JSON_FILE = "instanceMetadata.json";
+
+    private GraphUpdateUtils() {
+    }
+
+    public static void main(String[] args) {
+        if (args.length != 2) {
+            usage();
+            System.exit(1);
+        }
+        System.out.println(BANNER_MSG);
+        String operation = args[0].toLowerCase();
+        if (!(operation.equals(EXPORT) || operation.equals(IMPORT))) {
+            usage();
+            System.exit(1);
+        }
+        String utilsDir = args[1];
+        File utilsDirFile = new File(utilsDir);
+        if (!utilsDirFile.isDirectory()) {
+            System.err.println(utilsDir + " is not a valid directory");
+            System.exit(1);
+        }
+        String jsonFile = new File(utilsDirFile, INSTANCE_JSON_FILE).getAbsolutePath();
+        try {
+            Graph graph;
+            if (operation.equals(EXPORT)) {
+                graph = MetadataMappingService.initializeGraphDB();
+                GraphSONWriter.outputGraph(graph, jsonFile);
+                System.out.println("Exported instance metadata to " + jsonFile);
+            } else {
+                // Backup existing graphDB dir
+                Configuration graphConfig = MetadataMappingService.getConfiguration();
+                String graphStore = (String) graphConfig.getProperty("storage.directory");
+                File graphStoreFile = new File(graphStore);
+                File graphDirBackup = new File(graphStore + "_backup");
+                if (graphDirBackup.exists()) {
+                    FileUtils.deleteDirectory(graphDirBackup);
+                }
+                FileUtils.copyDirectory(graphStoreFile, graphDirBackup);
+
+                // delete graph dir first and then init graphDB to ensure IMPORT happens
into empty DB.
+                FileUtils.deleteDirectory(graphStoreFile);
+                graph = MetadataMappingService.initializeGraphDB();
+
+                // Import, if there is an exception restore backup.
+                try {
+                    GraphSONReader.inputGraph(graph, jsonFile);
+                    System.out.println("Imported instance metadata to " + jsonFile);
+                } catch (Exception ex) {
+                    String errorMsg = ex.getMessage();
+                    if (graphStoreFile.exists()) {
+                        FileUtils.deleteDirectory(graphStoreFile);
+                    }
+                    FileUtils.copyDirectory(graphDirBackup, graphStoreFile);
+                    throw new FalconException(errorMsg);
+                }
+            }
+        } catch (Exception e) {
+            System.err.println("Error " + operation + "ing JSON data to " + jsonFile + ",
" + e.getMessage());
+            e.printStackTrace(System.out);
+            System.exit(1);
+        }
+        System.exit(0);
+    }
+
+    public static void usage() {
+        StringBuilder usageMessage = new StringBuilder(1024);
+        usageMessage.append("usage: java ").append(GraphUpdateUtils.class.getName())
+                .append(" {").append(EXPORT).append('|').append(IMPORT).append("} <directory>");
+        System.err.println(usageMessage);
+    }
+}

http://git-wip-us.apache.org/repos/asf/falcon/blob/80529852/common/src/main/java/org/apache/falcon/metadata/MetadataMappingService.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/metadata/MetadataMappingService.java b/common/src/main/java/org/apache/falcon/metadata/MetadataMappingService.java
index 727be56..225e44a 100644
--- a/common/src/main/java/org/apache/falcon/metadata/MetadataMappingService.java
+++ b/common/src/main/java/org/apache/falcon/metadata/MetadataMappingService.java
@@ -139,14 +139,14 @@ public class MetadataMappingService
         }
     }
 
-    protected Graph initializeGraphDB() {
+    public static Graph initializeGraphDB() {
         LOG.info("Initializing graph db");
         Configuration graphConfig = getConfiguration();
         validateConfiguration(graphConfig);
         return GraphFactory.open(graphConfig);
     }
 
-    private void validateConfiguration(Configuration graphConfig) {
+    private static void validateConfiguration(Configuration graphConfig) {
         // check if storage backend if configured
         if (!graphConfig.containsKey(PROPERTY_KEY_STORAGE_BACKEND)) {
             throw new FalconRuntimException("Titan GraphDB storage backend is not configured.
"

http://git-wip-us.apache.org/repos/asf/falcon/blob/80529852/docs/src/site/twiki/InstallationSteps.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/InstallationSteps.twiki b/docs/src/site/twiki/InstallationSteps.twiki
index 93b1eab..297d88e 100644
--- a/docs/src/site/twiki/InstallationSteps.twiki
+++ b/docs/src/site/twiki/InstallationSteps.twiki
@@ -27,16 +27,15 @@ $ mvn clean install
 </verbatim>
 It builds and installs the package into the local repository, for use as a dependency in
other projects locally.
 
-[optionally -Dhadoop.version=<<hadoop.version>> can be appended to build for
a specific version of Hadoop]
-
-*NOTE:* Falcon drops support for Hadoop-1 and only supports Hadoop-2 from Falcon 0.6 onwards
-[optionally -Doozie.version=<<oozie version>> can be appended to build with a
specific version of Oozie. Oozie versions
->= 4 are supported]
-NOTE: Falcon builds with JDK 1.7/1.8 using -noverify option
-      To compile Falcon with Hive Replication, optionally "-P hadoop-2,hivedr" can be appended.
For this Hive >= 1.2.0
-      and Oozie >= 4.2.0 should be available.
+[optionally -Dhadoop.version=<<hadoop.version>> can be appended to build for
a specific version of hadoop]
 
+*Note 1:* Falcon drops support for Hadoop-1 and only supports Hadoop-2 from Falcon 0.6 onwards
+          Falcon build with JDK 1.7 using -noverify option
 
+*Note 2:* To compile Falcon with addon extensions, append additional profiles to build command
using syntax -P<<profile1,profile2>>
+          For Hive Mirroring extension, use profile"hivedr". Hive >= 1.2.0 and Oozie >=
4.2.0 is required
+          For HDFS Snapshot mirroring extension, use profile "hdfs-snapshot-mirroring". Hadoop
>= 2.7.0 is required
+          For ADF integration, use profile "adf"
 
 ---+++Step 3 - Package and Deploy Falcon
 

http://git-wip-us.apache.org/repos/asf/falcon/blob/80529852/docs/src/site/twiki/MigrationInstructions.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/MigrationInstructions.twiki b/docs/src/site/twiki/MigrationInstructions.twiki
index 7c0e027..a11dbc4 100644
--- a/docs/src/site/twiki/MigrationInstructions.twiki
+++ b/docs/src/site/twiki/MigrationInstructions.twiki
@@ -1,15 +1,32 @@
 ---+ Migration Instructions
 
----++ Migrate from 0.5-incubating to 0.6-incubating
+---++ Migrate from 0.9 to 0.10
 
-This is a placeholder wiki for migration instructions from falcon 0.5-incubating to 0.6-incubating.
+FALCON-1333 (Instance Search feature) requires Falcon to use titan-berkeleyje version 0.5.4
to support indexing.
+Up until version 0.9 - Falcon used titan-berkeleyje-jre6 version 0.4.2. GraphDB created by
version 0.4.2 cannot be
+read by version 0.5.4. The solution is to migrate the GraphDB to be compatible with Falcon
0.10 release. Please make
+sure that no falcon server is running while performing the migration.
 
----+++ Update Entities
+---+++ 1. Install Falcon 0.10
+Install Falcon 0.10 by following the [[InstallationSteps][Installation Steps]]. Do not start
the falcon server yet.
+The tool to migrate graphDB is packaged with 0.10 Falcon server in falcon-common-0.10.jar.
 
----+++ Change cluster dir permissions
+---+++ 2. Export GraphDB to JSON file using Falcon 0.9
+Please run the following command to generate the JSON file.
 
----+++ Enable/Disable TLS
+<verbatim>
+ $FALCON_HOME/bin/graphdbutil.sh export <<java_home> <<hadoop_home>>
<<falcon_0.9_home>> <<path_to_falcon-common-0.10.jar>> /jsonFile/dir/
+</verbatim>
 
----+++ Authorization
+This command will create /jsonFile/dir/instanceMetadata.json
 
+---+++ 3. Import GraphDB from JSON file using Falcon 0.10
+Please run the following command to import graphDB the JSON file. The location of graphDB
will be based on property
+"*.falcon.graph.storage.directory" set in startup.properties file.
+
+<verbatim>
+  $FALCON_HOME/bin/graphdbutil.sh export <<java_home> <<hadoop_home>>
<<falcon_0.10_home>> <<path_to_falcon-common-0.10.jar>> /jsonFile/dir/
+</verbatim>
+
+This command will import from /jsonFile/dir/instanceMetadata.json, now start the Falcon 0.10
server.
 

http://git-wip-us.apache.org/repos/asf/falcon/blob/80529852/release-docs/0.10/CHANGES.0.10.md
----------------------------------------------------------------------
diff --git a/release-docs/0.10/CHANGES.0.10.md b/release-docs/0.10/CHANGES.0.10.md
index 999af01..668dc97 100644
--- a/release-docs/0.10/CHANGES.0.10.md
+++ b/release-docs/0.10/CHANGES.0.10.md
@@ -54,6 +54,7 @@
 
 | JIRA | Summary | Priority | Component | Reporter | Contributor |
 |:---- |:---- | :--- |:---- |:---- |:---- |
+| [FALCON-2104](https://issues.apache.org/jira/browse/FALCON-2104) | Loss of data in GraphDB
when upgrading Falcon from 0.9 to 0.10. |  Blocker | . | Balu Vellanki | Balu Vellanki |
 | [FALCON-2100](https://issues.apache.org/jira/browse/FALCON-2100) | Remove dependency on
com.vividsolutions.jts |  Major | . | Balu Vellanki | Balu Vellanki |
 | [FALCON-2090](https://issues.apache.org/jira/browse/FALCON-2090) | HDFS Snapshot failed
with UnknownHostException when scheduling in HA Mode |  Critical | replication | Murali Ramasami
| Balu Vellanki |
 | [FALCON-2088](https://issues.apache.org/jira/browse/FALCON-2088) | Entity submission fails
with EntityNotRegisteredException in distributed mode |  Blocker | feed, prism, process |
Pragya Mittal | Praveen Adlakha |

http://git-wip-us.apache.org/repos/asf/falcon/blob/80529852/src/bin/graphdbutil.sh
----------------------------------------------------------------------
diff --git a/src/bin/graphdbutil.sh b/src/bin/graphdbutil.sh
new file mode 100644
index 0000000..151ec2f
--- /dev/null
+++ b/src/bin/graphdbutil.sh
@@ -0,0 +1,118 @@
+#!/bin/sh
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+#
+
+
+usage() {
+  echo "usage: $0  operation java-home hadoop-home falcon-home falcon-common-jar input/out-dir"
+  echo "  where operation is either export OR import"
+  echo "        java-home is the java installation location"
+  echo "        hadoop-home is the hadoop installation location"
+  echo "        falcon-home is the falcon home installation location"
+  echo "        falcon-common-jar is the falcon-common-<version>.jar location with
GraphUtils"
+  echo "        input/output dir is the directory for the graph data"
+  exit 1
+}
+
+if [ $# != 6 ]; then
+  usage
+fi
+
+operation=$1
+java_home=$2
+hadoop_home=$3
+falcon_home=$4
+falcon_common_jar=$5
+util_dir=$6
+
+export=0
+import=0
+keep_temp=Y
+
+case $operation in
+   import) import=1
+           ;;
+   export) export=1
+           ;;
+   *)     echo "Unknown operation $operation"
+          usage
+esac
+
+if [ -d  $java_home -a -f $java_home/bin/java -a -f $java_home/bin/jar ] ; then
+  :
+else
+  echo "Invalid java home directory $java_home"
+  usage
+fi
+
+if [ -d  $hadoop_home -a -f $hadoop_home/bin/hadoop ] ; then
+  :
+else
+  echo "Invalid hadoop home directory $hadoop_home"
+  usage
+fi
+
+if [ -d  $falcon_home -a -f $falcon_home/bin/falcon ] ; then
+  :
+else
+  echo "Invalid falcon home directory $falcon_home"
+  usage
+fi
+
+falcon_war=$falcon_home/server/webapp/falcon.war
+if [ ! -f $falcon_war ]; then
+  echo "Falcon war file $falcon_war not available"
+  usage
+fi
+
+if [ ! -f $falcon_common_jar ]; then
+  echo "Falcon commons jar file $falcon_common_jar not available"
+  usage
+fi
+
+
+util_tmpdir=/tmp/falcon-graphutil-tmp-$$
+echo "Using $util_tmpdir as temporary directory"
+trap "rm -rf $util.tmpdir" 0 2 3 15
+rm -rf $util_tmpdir
+mkdir -p $util_tmpdir
+
+if [ ! -d $util_dir ]; then
+   echo "Directory $util_dir does not exist"
+   usage
+fi
+
+if [ x$import = x1 ]; then
+   if [ ! -f $metadata_file ]; then
+      echo "Directory $util_dir does not exist or $metadata_file not present"
+      usage
+   fi
+fi
+
+cd $util_tmpdir
+jar -xf $falcon_war
+rm ./WEB-INF/lib/jackson*  ./WEB-INF/lib/falcon-common*.jar ./WEB-INF/lib/slf4j* ./WEB-INF/lib/activemq*
+cp $falcon_common_jar ./WEB-INF/lib/
+
+JAVA_HOME=$java_home
+export PATH=$JAVA_HOME/bin:$PATH
+export CLASSPATH="$falcon_home/conf:./WEB-INF/lib/*:`$hadoop_home/bin/hadoop classpath`"
+echo "Using classpath $CLASSPATH"
+java -Dfalcon.log.dir=/tmp/ org.apache.falcon.metadata.GraphUpdateUtils $operation $util_dir
+
+if [ x$keep_temp = xY ]; then
+  :
+else
+  rm -rf $util_tmpdir
+fi
\ No newline at end of file


Mime
View raw message