falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From srik...@apache.org
Subject [1/3] FALCON-66 Make oozie version change configurable. Contributed by Shwetha GS
Date Tue, 10 Dec 2013 05:12:23 GMT
Updated Branches:
  refs/heads/master 3825357b8 -> 73e54e04b


http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/73e54e04/oozie/src/main/java/org/apache/falcon/workflow/engine/NullBundleJob.java
----------------------------------------------------------------------
diff --git a/oozie/src/main/java/org/apache/falcon/workflow/engine/NullBundleJob.java b/oozie/src/main/java/org/apache/falcon/workflow/engine/NullBundleJob.java
index aead7eb..c1f10d9 100644
--- a/oozie/src/main/java/org/apache/falcon/workflow/engine/NullBundleJob.java
+++ b/oozie/src/main/java/org/apache/falcon/workflow/engine/NullBundleJob.java
@@ -126,7 +126,6 @@ public class NullBundleJob implements BundleJob {
         return null;
     }
 
-    @Override
     public String getAcl() {
         return null;
     }

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/73e54e04/oozie/src/main/java/org/apache/falcon/workflow/engine/NullCoordJob.java
----------------------------------------------------------------------
diff --git a/oozie/src/main/java/org/apache/falcon/workflow/engine/NullCoordJob.java b/oozie/src/main/java/org/apache/falcon/workflow/engine/NullCoordJob.java
deleted file mode 100644
index 83314e3..0000000
--- a/oozie/src/main/java/org/apache/falcon/workflow/engine/NullCoordJob.java
+++ /dev/null
@@ -1,158 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.falcon.workflow.engine;
-
-import org.apache.oozie.client.CoordinatorAction;
-import org.apache.oozie.client.CoordinatorJob;
-
-import java.util.Date;
-import java.util.List;
-
-/**
- * Default coord job.
- */
-public class NullCoordJob implements CoordinatorJob {
-
-    @Override
-    public String getAppPath() {
-        return null;
-    }
-
-    @Override
-    public String getAppName() {
-        return null;
-    }
-
-    @Override
-    public String getId() {
-        return null;
-    }
-
-    @Override
-    public String getConf() {
-        return null;
-    }
-
-    @Override
-    public Status getStatus() {
-        return null;
-    }
-
-    @Override
-    public String getFrequency() {
-        return null;
-    }
-
-    @Override
-    public Timeunit getTimeUnit() {
-        return null;
-    }
-
-    @Override
-    public String getTimeZone() {
-        return null;
-    }
-
-    @Override
-    public int getConcurrency() {
-        return 0;
-    }
-
-    @Override
-    public Execution getExecutionOrder() {
-        return null;
-    }
-
-    @Override
-    public int getTimeout() {
-        return 0;
-    }
-
-    @Override
-    public Date getLastActionTime() {
-        return null;
-    }
-
-    @Override
-    public Date getNextMaterializedTime() {
-        return null;
-    }
-
-    @Override
-    public Date getStartTime() {
-        return null;
-    }
-
-    @Override
-    public Date getEndTime() {
-        return null;
-    }
-
-    @Override
-    public void setStatus(Status status) {
-    }
-
-    @Override
-    public void setPending() {
-    }
-
-    @Override
-    public void resetPending() {
-    }
-
-    @Override
-    public Date getPauseTime() {
-        return null;
-    }
-
-    @Override
-    public String getExternalId() {
-        return null;
-    }
-
-    @Override
-    public String getUser() {
-        return null;
-    }
-
-    @Override
-    public String getGroup() {
-        return null;
-    }
-
-    @Override
-    public String getBundleId() {
-        return null;
-    }
-
-    @Override
-    public String getConsoleUrl() {
-        return null;
-    }
-
-    @Override
-    public List<CoordinatorAction> getActions() {
-        return null;
-    }
-
-    @Override
-    public String getAcl() {
-        return null;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/73e54e04/oozie/src/main/java/org/apache/falcon/workflow/engine/OozieWorkflowEngine.java
----------------------------------------------------------------------
diff --git a/oozie/src/main/java/org/apache/falcon/workflow/engine/OozieWorkflowEngine.java
b/oozie/src/main/java/org/apache/falcon/workflow/engine/OozieWorkflowEngine.java
index a6ecf3e..c4c46af 100644
--- a/oozie/src/main/java/org/apache/falcon/workflow/engine/OozieWorkflowEngine.java
+++ b/oozie/src/main/java/org/apache/falcon/workflow/engine/OozieWorkflowEngine.java
@@ -704,7 +704,7 @@ public class OozieWorkflowEngine extends AbstractWorkflowEngine {
             List<CoordinatorAction> actions = new ArrayList<CoordinatorAction>();
 
             for (CoordinatorJob coord : applicableCoords) {
-                Frequency freq = createFrequency(coord.getFrequency(), coord.getTimeUnit());
+                Frequency freq = createFrequency(String.valueOf(coord.getFrequency()), coord.getTimeUnit());
                 TimeZone tz = EntityUtil.getTimeZone(coord.getTimeZone());
                 Date iterStart = EntityUtil.getNextStartTime(coord.getStartTime(), freq,
tz, start);
                 Date iterEnd = (coord.getNextMaterializedTime().before(end) ? coord.getNextMaterializedTime()
: end);
@@ -722,7 +722,7 @@ public class OozieWorkflowEngine extends AbstractWorkflowEngine {
                     }
                     Calendar startCal = Calendar.getInstance(EntityUtil.getTimeZone(coord.getTimeZone()));
                     startCal.setTime(iterStart);
-                    startCal.add(freq.getTimeUnit().getCalendarUnit(), Integer.parseInt(coord.getFrequency()));
+                    startCal.add(freq.getTimeUnit().getCalendarUnit(), Integer.valueOf((coord.getFrequency())));
                     iterStart = startCal.getTime();
                 }
             }
@@ -890,8 +890,7 @@ public class OozieWorkflowEngine extends AbstractWorkflowEngine {
             Calendar cal = Calendar.getInstance(EntityUtil.getTimeZone(coord
                     .getTimeZone()));
             cal.setTime(coord.getLastActionTime());
-            Frequency freq = createFrequency(coord.getFrequency(),
-                    coord.getTimeUnit());
+            Frequency freq = createFrequency(String.valueOf(coord.getFrequency()), coord.getTimeUnit());
             cal.add(freq.getTimeUnit().getCalendarUnit(), -freq.getFrequencyAsInt());
             return cal.getTime();
         }

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/73e54e04/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index e1b2e32..aa805cf 100644
--- a/pom.xml
+++ b/pom.xml
@@ -100,7 +100,8 @@
         <hadoop.version>1.1.2</hadoop.version>
         <slf4j.version>1.6.1</slf4j.version>
         <oozie.version>4.0.0</oozie.version>
-        <falcon.oozie.version>${oozie.version}-falcon</falcon.oozie.version>
+        <oozie.buildversion>${oozie.version}-falcon</oozie.buildversion>
+        <oozie.forcebuild>false</oozie.forcebuild>
         <activemq.version>5.4.3</activemq.version>
         <hive.version>0.11.0</hive.version>
         <hcatalog.version>0.11.0</hcatalog.version>
@@ -144,6 +145,7 @@
     </profiles>
 
     <modules>
+        <module>checkstyle</module>
         <module>build-tools</module>
         <module>client</module>
         <module>metrics</module>
@@ -526,7 +528,7 @@
             <dependency>
                 <groupId>org.apache.oozie</groupId>
                 <artifactId>oozie-client</artifactId>
-                <version>${falcon.oozie.version}</version>
+                <version>${oozie.buildversion}</version>
                 <exclusions>
                     <exclusion>
                         <groupId>org.apache.hadoop</groupId>
@@ -562,7 +564,7 @@
             <dependency>
                 <groupId>org.apache.oozie</groupId>
                 <artifactId>oozie-core</artifactId>
-                <version>${falcon.oozie.version}</version>
+                <version>${oozie.buildversion}</version>
                 <exclusions>
                     <exclusion>
                         <groupId>org.apache.hadoop</groupId>
@@ -618,7 +620,7 @@
             <dependency>
                 <groupId>org.apache.oozie</groupId>
                 <artifactId>oozie-webapp</artifactId>
-                <version>${falcon.oozie.version}</version>
+                <version>${oozie.buildversion}</version>
                 <type>war</type>
             </dependency>
 
@@ -741,18 +743,6 @@
             </dependency>
 
             <dependency>
-                <groupId>org.apache.hive</groupId>
-                <artifactId>hive-metastore</artifactId>
-                <version>${hive.version}</version>
-                <exclusions>
-                    <exclusion> <!--Oozie already imports this-->
-                        <groupId>org.apache.derby</groupId>
-                        <artifactId>derby</artifactId>
-                    </exclusion>
-                </exclusions>
-            </dependency>
-
-            <dependency>
                 <groupId>org.apache.hcatalog</groupId>
                 <artifactId>webhcat-java-client</artifactId>
                 <version>${hcatalog.version}</version>
@@ -763,6 +753,12 @@
                 <artifactId>annotations</artifactId>
                 <version>1.3.2</version>
             </dependency>
+
+            <dependency>
+                <groupId>xerces</groupId>
+                <artifactId>xercesImpl</artifactId>
+                <version>2.10.0</version>
+            </dependency>
         </dependencies>
     </dependencyManagement>
 
@@ -889,6 +885,30 @@
 
         <plugins>
             <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-enforcer-plugin</artifactId>
+                <version>1.3.1</version>
+                <executions>
+                    <execution>
+                        <id>enforce-property</id>
+                        <goals>
+                            <goal>enforce</goal>
+                        </goals>
+                        <configuration>
+                            <rules>
+                                <requireProperty>
+                                    <property>oozie.version</property>
+                                    <regex>^(?:(?!3\.1\.3-incubating).)*$</regex>
+                                    <regexMessage>Oozie version 3.1.3-incubating is
not supported. Use a higher version of oozie</regexMessage>
+                                </requireProperty>
+                            </rules>
+                            <fail>true</fail>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+
+            <plugin>
                 <groupId>org.codehaus.mojo</groupId>
                 <artifactId>buildnumber-maven-plugin</artifactId>
                 <executions>
@@ -1008,7 +1028,7 @@
                         <exclude>**/target/**</exclude>
                         <exclude>**/activemq-data/**</exclude>
                         <exclude>**/build/**</exclude>
-                        <exclude>*.patch</exclude>
+                        <exclude>**/*.patch</exclude>
                         <exclude>derby.log</exclude>
                         <exclude>**/logs/**</exclude>
                         <exclude>**/.classpath</exclude>
@@ -1020,7 +1040,6 @@
                         <exclude>**/.externalToolBuilders/**</exclude>
                         <exclude>html5-ui/**</exclude>
                     </excludes>
-                    <argLine>-Dfalcon.log.dir=target/logs -Dfalcon.embeddedmq.data=target/data</argLine>
                 </configuration>
                 <executions>
                     <execution>
@@ -1039,7 +1058,7 @@
                 <dependencies>
                     <dependency>
                         <groupId>org.apache.falcon</groupId>
-                        <artifactId>build-tools</artifactId>
+                        <artifactId>checkstyle</artifactId>
                         <version>${project.version}</version>
                     </dependency>
                 </dependencies>
@@ -1067,7 +1086,7 @@
                 <configuration>
                     <!--debug>true</debug -->
                     <xmlOutput>true</xmlOutput>
-                    <excludeFilterFile>${basedir}/../build-tools/src/main/resources/falcon/findbugs-exclude.xml</excludeFilterFile>
+                    <excludeFilterFile>${basedir}/../checkstyle/src/main/resources/falcon/findbugs-exclude.xml</excludeFilterFile>
                     <failOnError>true</failOnError>
                     <skip>${skipCheck}</skip>
                 </configuration>

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/73e54e04/process/pom.xml
----------------------------------------------------------------------
diff --git a/process/pom.xml b/process/pom.xml
index 88505fd..912c8c0 100644
--- a/process/pom.xml
+++ b/process/pom.xml
@@ -107,5 +107,10 @@
             <groupId>org.testng</groupId>
             <artifactId>testng</artifactId>
         </dependency>
+
+        <dependency>
+            <groupId>xerces</groupId>
+            <artifactId>xercesImpl</artifactId>
+        </dependency>
     </dependencies>
 </project>

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/73e54e04/src/bin/package.sh
----------------------------------------------------------------------
diff --git a/src/bin/package.sh b/src/bin/package.sh
index 0946e9a..3a8ebee 100755
--- a/src/bin/package.sh
+++ b/src/bin/package.sh
@@ -16,9 +16,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-if [ "${1}"x == "x" ]
+set -e
+set -x
+
+if [ $# -ne 2 ]
 then
-  echo "Usage ${0} <<hadoop-version>>"
+  echo "Usage ${0} <<hadoop-version>> <<oozie-version>>"
   exit 1
 fi
 
@@ -39,30 +42,27 @@ BASEDIR=`dirname ${PRG}`
 BASEDIR=`cd ${BASEDIR};pwd`
 
 FALCON_SRC=${BASEDIR}/../..
-PACKAGE_HOME=${FALCON_SRC}/target/package
-
-pushd ${FALCON_SRC}
-echo "Building falcon & falcon-oozie-el-extensions ....."
-mvn clean assembly:assembly -Dhadoop.version=${1} -DskipTests -DskipCheck=true > /dev/null
-popd
 
-mkdir -p ${PACKAGE_HOME}
-pushd ${PACKAGE_HOME}
-rm -rf oozie-*
-echo "Getting oozie release tar ball of version 4.0.0 ..."
-curl "http://www.apache.org/dist/oozie/4.0.0/oozie-4.0.0.tar.gz" -o oozie-4.0.0.tgz
-tar -xzvf oozie-4.0.0.tgz 2> /dev/null
-rm oozie-4.0.0.tgz
-cd oozie-4.0.0
+HADOOP_VERSION=$1
+OOZIE_VERSION=$2
 
-echo "Patching oozie with falcon extensions and marking version as 4.0.0 ..."
-patch -p0 < ${FALCON_SRC}/build-tools/src/patch/oozie-1551-hadoop-2-profile.patch
-patch -p0 < ${FALCON_SRC}/build-tools/src/patch/oozie-4.0.0-falcon.patch
-patch -p0 < ${FALCON_SRC}/build-tools/src/patch/oozie-bundle-el-extension.patch
+pushd ${FALCON_SRC}
+echo "Builing oozie-el-extension and oozie"
+mvn clean install -pl build-tools,hadoop-dependencies,oozie-el-extensions -am -Dhadoop.version=$HADOOP_VERSION
-Doozie.version=$OOZIE_VERSION -Doozie.forcebuild=true -DskipTests
+pushd target/oozie-$OOZIE_VERSION
+bin/mkdistro.sh -DskipTests
+pushd distro/target/oozie-*
+mkdir -p WEB-INF/lib
+cp ${FALCON_SRC}/oozie-el-extensions/target/falcon-oozie-el-extension*.jar WEB-INF/lib/
+jar uvf oozie-*/oozie.war WEB-INF/lib/*.jar
+mkdir libext
+cp ${FALCON_SRC}/hadoop-dependencies/target/dependency/*.jar libext
+tar -zcvf ${FALCON_SRC}/target/oozie-$OOZIE_VERSION-distro.tar.gz oozie-*
 
-echo "Building oozie & creating tar ball ..."
-bin/mkdistro.sh -DskipTests > /dev/null
+popd
+popd
+mvn assembly:assembly -Dhadoop.version=$HADOOP_VERSION -Doozie.version=$OOZIE_VERSION -Doozie.forcebuild=true
-DskipTests -DskipCheck=true
 
-echo "Falcon pacakge is available in ${FALCON_SRC}/target/falcon-<<version>>/falcon-<<version>>.tar.gz"
-echo "Oozie pacakge is available in ${FALCON_SRC}/target/package/oozie-4.0.0/distro/target/oozie-4.0.0-distro.tar.gz"
+echo "Falcon pacakge is available in ${FALCON_SRC}/target/falcon-<<version>>-bin.tar.gz"
+echo "Oozie pacakge is available in ${FALCON_SRC}/target/oozie-$OOZIE_VERSION-distro.tar.gz"
 popd

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/73e54e04/src/bin/service-start.sh
----------------------------------------------------------------------
diff --git a/src/bin/service-start.sh b/src/bin/service-start.sh
index d4950d5..430bb1a 100755
--- a/src/bin/service-start.sh
+++ b/src/bin/service-start.sh
@@ -43,7 +43,7 @@ mkdir -p $FALCON_LOG_DIR
 
 pushd ${BASEDIR} > /dev/null
 
-JAVA_PROPERTIES="$FALCON_OPTS $FALCON_PROPERTIES -Dfalcon.log.dir=$FALCON_LOG_DIR -Dfalcon.embeddedmq.data=$FALCON_DATA_DIR
-Dfalcon.home=${FALCON_HOME_DIR} -Dconfig.location=$FALCON_CONF -Dfalcon.app.type=$APP_TYPE"
+JAVA_PROPERTIES="$FALCON_OPTS $FALCON_PROPERTIES -Dfalcon.log.dir=$FALCON_LOG_DIR -Dfalcon.embeddedmq.data=$FALCON_DATA_DIR
-Dfalcon.home=${FALCON_HOME_DIR} -Dconfig.location=$FALCON_CONF -Dfalcon.app.type=$APP_TYPE
-Dfalcon.catalog.service.enabled=$CATALOG_ENABLED"
 shift
 
 while [[ ${1} =~ ^\-D ]]; do

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/73e54e04/src/conf/startup.properties
----------------------------------------------------------------------
diff --git a/src/conf/startup.properties b/src/conf/startup.properties
index 3d55aee..69613f6 100644
--- a/src/conf/startup.properties
+++ b/src/conf/startup.properties
@@ -48,7 +48,7 @@ prism.configstore.listeners=org.apache.falcon.entity.v0.EntityGraph,\
                         org.apache.falcon.entity.ColoClusterRelation,\
                         org.apache.falcon.group.FeedGroupMap
 *.broker.impl.class=org.apache.activemq.ActiveMQConnectionFactory
-*.shared.libs=activemq-core,ant,geronimo-j2ee-management,hadoop-distcp,jms,json-simple,oozie-client,spring-jms,s4fs-0.1.jar
+*.shared.libs=activemq-core,ant,geronimo-j2ee-management,hadoop-distcp,jms,json-simple,oozie-client,spring-jms
 
 ######### Implementation classes #########
 

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/73e54e04/webapp/pom.xml
----------------------------------------------------------------------
diff --git a/webapp/pom.xml b/webapp/pom.xml
index acdd857..f3d52c1 100644
--- a/webapp/pom.xml
+++ b/webapp/pom.xml
@@ -285,7 +285,7 @@
                         <goals>
                             <goal>run</goal>
                         </goals>
-                        <phase>process-resources</phase>
+                        <phase>pre-integration-test</phase>
                     </execution>
                 </executions>
             </plugin>

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/73e54e04/webapp/src/conf/oozie/conf/oozie-log4j.properties
----------------------------------------------------------------------
diff --git a/webapp/src/conf/oozie/conf/oozie-log4j.properties b/webapp/src/conf/oozie/conf/oozie-log4j.properties
new file mode 100644
index 0000000..34e1634
--- /dev/null
+++ b/webapp/src/conf/oozie/conf/oozie-log4j.properties
@@ -0,0 +1,43 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License. See accompanying LICENSE file.
+#
+
+# If the Java System property 'oozie.log.dir' is not defined at Oozie start up time
+# XLogService sets its value to '${oozie.home}/logs'
+
+log4j.appender.oozie=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.oozie.DatePattern='.'yyyy-MM-dd-HH
+
+log4j.appender.oozie.File=${oozie.log.dir}/oozie.log
+log4j.appender.oozie.Append=true
+log4j.appender.oozie.layout=org.apache.log4j.PatternLayout
+log4j.appender.oozie.layout.ConversionPattern=%d{ISO8601} %5p %c{1}:%L - %m%n
+
+log4j.logger.org.apache.oozie=DEBUG, oozie
+log4j.logger.org.apache.hadoop=WARN, oozie
+log4j.logger.org.mortbay=WARN, oozie
+log4j.logger.org.hsqldb=WARN, oozie
+log4j.logger.org.apache.hadoop.security.authentication.server=DEBUG, oozie

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/73e54e04/webapp/src/conf/oozie/conf/oozie-site.xml
----------------------------------------------------------------------
diff --git a/webapp/src/conf/oozie/conf/oozie-site.xml b/webapp/src/conf/oozie/conf/oozie-site.xml
index 84eaa61..48408ba 100644
--- a/webapp/src/conf/oozie/conf/oozie-site.xml
+++ b/webapp/src/conf/oozie/conf/oozie-site.xml
@@ -22,7 +22,6 @@
         Refer to the oozie-default.xml file for the complete list of
         Oozie configuration properties and their default values.
     -->
-
     <property>
         <name>oozie.service.ActionService.executor.ext.classes</name>
         <value>


Mime
View raw message