falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From b...@apache.org
Subject falcon git commit: FALCON-1905 Extension Job Management: IT tests for REST APIs and CLIs
Date Thu, 12 May 2016 18:34:51 GMT
Repository: falcon
Updated Branches:
  refs/heads/master 98a904c21 -> 7b78c39eb


FALCON-1905 Extension Job Management: IT tests for REST APIs and CLIs

Author: yzheng-hortonworks <yzheng@hortonworks.com>

Reviewers: " Sowmya<sramesh@hortonworks.com>,Balu Vellanki <balu@apache.org>"

Closes #129 from yzheng-hortonworks/FALCON-1905


Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/7b78c39e
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/7b78c39e
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/7b78c39e

Branch: refs/heads/master
Commit: 7b78c39ebbddc6ff370e2c758c851ac22225a991
Parents: 98a904c
Author: yzheng-hortonworks <yzheng@hortonworks.com>
Authored: Thu May 12 11:34:37 2016 -0700
Committer: bvellanki <bvellanki@hortonworks.com>
Committed: Thu May 12 11:34:37 2016 -0700

----------------------------------------------------------------------
 .../falcon/resource/ExtensionInstanceList.java  |   9 +-
 .../falcon/resource/ExtensionJobList.java       |   9 +-
 .../falcon/extensions/store/ExtensionStore.java |   2 +-
 .../apache/falcon/extensions/ExtensionTest.java |   1 +
 .../store/AbstractTestExtensionStore.java       |  23 ++-
 .../extensions/store/ExtensionStoreTest.java    |   1 +
 webapp/pom.xml                                  |   8 +
 .../falcon/resource/ExtensionManagerIT.java     | 171 +++++++++++++++++++
 .../org/apache/falcon/resource/TestContext.java |  53 +++++-
 .../hdfs-mirroring-property-template.txt        |   9 +
 .../test/resources/hdfs-mirroring-template.xml  |  45 +++++
 .../hdfs-snapshot-mirroring-template.xml        |  45 +++++
 .../test/resources/hive-mirroring-template.xml  |  45 +++++
 webapp/src/test/resources/startup.properties    |   1 +
 14 files changed, 398 insertions(+), 24 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/falcon/blob/7b78c39e/client/src/main/java/org/apache/falcon/resource/ExtensionInstanceList.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/falcon/resource/ExtensionInstanceList.java b/client/src/main/java/org/apache/falcon/resource/ExtensionInstanceList.java
index 65ca4d4..a7ca4e4 100644
--- a/client/src/main/java/org/apache/falcon/resource/ExtensionInstanceList.java
+++ b/client/src/main/java/org/apache/falcon/resource/ExtensionInstanceList.java
@@ -28,14 +28,15 @@ import java.util.List;
 /**
  * Instance list of an extension job used for marshalling / unmarshalling with REST calls.
  */
+//SUSPEND CHECKSTYLE CHECK VisibilityModifierCheck
 @XmlRootElement
 @edu.umd.cs.findbugs.annotations.SuppressWarnings({"EI_EXPOSE_REP", "EI_EXPOSE_REP2"})
 public class ExtensionInstanceList {
     @XmlElement
-    private int numEntities;
+    public int numEntities;
 
     @XmlElementWrapper(name = "entitiesSummary")
-    private List<EntitySummary> entitySummary;
+    public List<EntitySummary> entitySummary;
 
     public ExtensionInstanceList() {
         numEntities = 0;
@@ -70,10 +71,10 @@ public class ExtensionInstanceList {
      */
     public static class EntitySummary {
         @XmlElement
-        private EntityList.EntityElement entityProfile;
+        public EntityList.EntityElement entityProfile;
 
         @XmlElement
-        private InstancesResult.Instance[] instances;
+        public InstancesResult.Instance[] instances;
 
         public EntitySummary() {
             entityProfile = null;

http://git-wip-us.apache.org/repos/asf/falcon/blob/7b78c39e/client/src/main/java/org/apache/falcon/resource/ExtensionJobList.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/falcon/resource/ExtensionJobList.java b/client/src/main/java/org/apache/falcon/resource/ExtensionJobList.java
index 71f92b8..ca031b5 100644
--- a/client/src/main/java/org/apache/falcon/resource/ExtensionJobList.java
+++ b/client/src/main/java/org/apache/falcon/resource/ExtensionJobList.java
@@ -27,15 +27,16 @@ import java.util.List;
 /**
  * Extension job list used for marshalling / unmarshalling with REST calls.
  */
+//SUSPEND CHECKSTYLE CHECK VisibilityModifierCheck
 @XmlRootElement
 @edu.umd.cs.findbugs.annotations.SuppressWarnings({"EI_EXPOSE_REP", "EI_EXPOSE_REP2"})
 public class ExtensionJobList {
 
     @XmlElement
-    private int numJobs;
+    public int numJobs;
 
     @XmlElementWrapper(name = "jobs")
-    private List<JobElement> job;
+    public List<JobElement> job;
 
     public ExtensionJobList() {
         numJobs = 0;
@@ -71,10 +72,10 @@ public class ExtensionJobList {
      */
     public static class JobElement {
         @XmlElement
-        private String jobName;
+        public String jobName;
 
         @XmlElement
-        private EntityList jobEntities;
+        public EntityList jobEntities;
 
         public JobElement() {
             jobName = null;

http://git-wip-us.apache.org/repos/asf/falcon/blob/7b78c39e/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java
----------------------------------------------------------------------
diff --git a/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java
b/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java
index 2c2e3c6..4ca432a 100644
--- a/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java
+++ b/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java
@@ -56,7 +56,7 @@ public final class ExtensionStore {
     private static final String RESOURCES_DIR = "resources";
     private static final String LIBS_DIR = "libs";
 
-    private static final String EXTENSION_STORE_URI = "extension.store.uri";
+    public static final String EXTENSION_STORE_URI = "extension.store.uri";
 
     private static final ExtensionStore STORE = new ExtensionStore();
 

http://git-wip-us.apache.org/repos/asf/falcon/blob/7b78c39e/extensions/src/test/java/org/apache/falcon/extensions/ExtensionTest.java
----------------------------------------------------------------------
diff --git a/extensions/src/test/java/org/apache/falcon/extensions/ExtensionTest.java b/extensions/src/test/java/org/apache/falcon/extensions/ExtensionTest.java
index b14d500..21e78d8 100644
--- a/extensions/src/test/java/org/apache/falcon/extensions/ExtensionTest.java
+++ b/extensions/src/test/java/org/apache/falcon/extensions/ExtensionTest.java
@@ -148,6 +148,7 @@ public class ExtensionTest extends AbstractTestExtensionStore {
 
     @BeforeClass
     public void init() throws Exception {
+        initExtensionStore();
         extension = new Extension();
         baseDir = Files.createTempDirectory("test_extensions_hdfs").toFile().getAbsoluteFile();
         miniDFSCluster = MiniHdfsClusterUtil.initMiniDfs(MiniHdfsClusterUtil.EXTENSION_TEST_PORT,
baseDir);

http://git-wip-us.apache.org/repos/asf/falcon/blob/7b78c39e/extensions/src/test/java/org/apache/falcon/extensions/store/AbstractTestExtensionStore.java
----------------------------------------------------------------------
diff --git a/extensions/src/test/java/org/apache/falcon/extensions/store/AbstractTestExtensionStore.java
b/extensions/src/test/java/org/apache/falcon/extensions/store/AbstractTestExtensionStore.java
index b62b475..fba803f 100644
--- a/extensions/src/test/java/org/apache/falcon/extensions/store/AbstractTestExtensionStore.java
+++ b/extensions/src/test/java/org/apache/falcon/extensions/store/AbstractTestExtensionStore.java
@@ -27,7 +27,6 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.testng.annotations.AfterClass;
-import org.testng.annotations.BeforeClass;
 
 import java.io.File;
 import java.io.IOException;
@@ -43,19 +42,19 @@ public class AbstractTestExtensionStore {
     protected ExtensionStore store;
     private FileSystem fileSystem;
 
-    @BeforeClass
-    public void initConfigStore() throws Exception {
-        String configPath = new URI(StartupProperties.get().getProperty("extension.store.uri")).getPath();
-        extensionStorePath = configPath + "-" + getClass().getName();
-        StartupProperties.get().setProperty("extension.store.uri", extensionStorePath);
+    public void initExtensionStore() throws Exception {
+        initExtensionStore(this.getClass());
+    }
+
+    public void initExtensionStore(Class resourceClass) throws Exception {
         new ExtensionService().init();
         store = ExtensionService.getExtensionStore();
         fileSystem = HadoopClientFactory.get().createFalconFileSystem(new Configuration(true));
-
-        extensionStoreSetup();
+        extensionStorePath = new URI(StartupProperties.get().getProperty(ExtensionStore.EXTENSION_STORE_URI)).getPath();
+        extensionStoreSetup(resourceClass);
     }
 
-    private void extensionStoreSetup() throws IOException {
+    private void extensionStoreSetup(Class resourceClass) throws IOException {
         List<AbstractExtension> extensions = AbstractExtension.getExtensions();
         for (AbstractExtension extension : extensions) {
             String extensionName = extension.getName().toLowerCase();
@@ -86,13 +85,13 @@ public class AbstractTestExtensionStore {
             Path dstFile = new Path(runTimeResourcePath, extensionName + "-template.xml");
             fileSystem.create(dstFile);
             String srcFile = extensionName + "-template.xml";
-            fileSystem.copyFromLocalFile(new Path(getAbsolutePath(srcFile)), dstFile);
+            fileSystem.copyFromLocalFile(new Path(getAbsolutePath(resourceClass, srcFile)),
dstFile);
         }
 
     }
 
-    private String getAbsolutePath(String fileName) {
-        return this.getClass().getResource("/" + fileName).getPath();
+    private String getAbsolutePath(Class resourceClass, String fileName) {
+        return resourceClass.getResource("/" + fileName).getPath();
     }
 
 

http://git-wip-us.apache.org/repos/asf/falcon/blob/7b78c39e/extensions/src/test/java/org/apache/falcon/extensions/store/ExtensionStoreTest.java
----------------------------------------------------------------------
diff --git a/extensions/src/test/java/org/apache/falcon/extensions/store/ExtensionStoreTest.java
b/extensions/src/test/java/org/apache/falcon/extensions/store/ExtensionStoreTest.java
index 9dbacde..bc04d88 100644
--- a/extensions/src/test/java/org/apache/falcon/extensions/store/ExtensionStoreTest.java
+++ b/extensions/src/test/java/org/apache/falcon/extensions/store/ExtensionStoreTest.java
@@ -35,6 +35,7 @@ public class ExtensionStoreTest extends AbstractTestExtensionStore {
 
     @BeforeClass
     public void init() throws Exception {
+        initExtensionStore();
         resourcesMap = ImmutableMap.of(
                 "hdfs-mirroring-template.xml", extensionStorePath
                         + "/hdfs-mirroring/resources/runtime/hdfs-mirroring-template.xml",

http://git-wip-us.apache.org/repos/asf/falcon/blob/7b78c39e/webapp/pom.xml
----------------------------------------------------------------------
diff --git a/webapp/pom.xml b/webapp/pom.xml
index 564a04c..ca1ff18 100644
--- a/webapp/pom.xml
+++ b/webapp/pom.xml
@@ -172,6 +172,14 @@
 
         <dependency>
             <groupId>org.apache.falcon</groupId>
+            <artifactId>falcon-extensions</artifactId>
+            <version>${project.version}</version>
+            <type>test-jar</type>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.falcon</groupId>
             <artifactId>falcon-retention</artifactId>
         </dependency>
 

http://git-wip-us.apache.org/repos/asf/falcon/blob/7b78c39e/webapp/src/test/java/org/apache/falcon/resource/ExtensionManagerIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/falcon/resource/ExtensionManagerIT.java b/webapp/src/test/java/org/apache/falcon/resource/ExtensionManagerIT.java
new file mode 100644
index 0000000..b352be4
--- /dev/null
+++ b/webapp/src/test/java/org/apache/falcon/resource/ExtensionManagerIT.java
@@ -0,0 +1,171 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.falcon.resource;
+
+import org.apache.falcon.extensions.ExtensionProperties;
+import org.apache.falcon.extensions.mirroring.hdfs.HdfsMirroringExtensionProperties;
+import org.apache.falcon.extensions.store.AbstractTestExtensionStore;
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Unit tests for org.apache.falcon.extensions.ExtensionManager.
+ */
+public class ExtensionManagerIT extends AbstractTestExtensionStore {
+    private static final String HDFS_MIRRORING_PROPERTY_TEMPLATE = "/hdfs-mirroring-property-template.txt";
+    private static final String JOB_NAME_1 = "hdfs-mirroring-job-1";
+    private static final String JOB_NAME_2 = "hdfs-mirroring-job-2";
+    private static final String CLUSTER_NAME = "primaryCluster";
+    private static final String START_TIME_1 = "2016-05-03T00:00Z";
+    private static final String START_TIME_2 = "2016-05-01T00:00Z";
+    private static final String FREQUENCY = "days(1)";
+    private static final String SOURCE_DIR = "/apps/falcon/demo/input-{year}-{month}-{day}";
+    private static final String TARGET_DIR = "/apps/falcon/demo/output-{year}-{month}-{day}";
+
+    private final TestContext context = new TestContext();
+
+    @BeforeClass
+    public void prepare() throws Exception {
+        TestContext.prepare();
+        initExtensionStore(TestContext.class);
+    }
+
+    @AfterClass
+    public void tearDown() throws Exception {
+        TestContext.deleteEntitiesFromStore();
+    }
+
+    @Test
+    public void testTrustedExtensionJob() throws Exception {
+        Map<String, String> overlay = context.getUniqueOverlay();
+        String endTime = context.getProcessEndTime();
+
+        // submit cluster
+        overlay.put("cluster", CLUSTER_NAME);
+        String filePath = TestContext.overlayParametersOverTemplate(TestContext.CLUSTER_TEMPLATE,
overlay);
+        System.out.println("entity -submit -type cluster -file " + filePath);
+        Assert.assertEquals(TestContext.executeWithURL("entity -submit -type cluster -file
" + filePath), 0);
+
+        // submit
+        System.out.println(getHDFSMirroringProperty(JOB_NAME_1, START_TIME_1, endTime).toString());
+        filePath = TestContext.overlayParametersOverTemplate(
+                TestContext.getTempFile("property", "txt"),
+                HDFS_MIRRORING_PROPERTY_TEMPLATE,
+                getHDFSMirroringProperty(JOB_NAME_1, START_TIME_1, endTime));
+        System.out.println("extension -submit -extensionName hdfs-mirroring -file " + filePath);
+        Assert.assertEquals(TestContext.executeWithURL(
+                "extension -submit -extensionName hdfs-mirroring -file " + filePath), 0);
+
+        // schedule
+        System.out.println("extension -schedule -jobName " + JOB_NAME_1);
+        Assert.assertEquals(TestContext.executeWithURL("extension -schedule -jobName " +
JOB_NAME_1), 0);
+
+        // submit and schedule
+        filePath = TestContext.overlayParametersOverTemplate(
+                TestContext.getTempFile("property", "txt"),
+                HDFS_MIRRORING_PROPERTY_TEMPLATE,
+                getHDFSMirroringProperty(JOB_NAME_2, START_TIME_1, endTime));
+        System.out.println("extension -submitAndSchedule -extensionName hdfs-mirroring -file
" + filePath);
+        Assert.assertEquals(TestContext.executeWithURL(
+                "extension -submitAndSchedule -extensionName hdfs-mirroring -file " + filePath),
0);
+
+        // list extension jobs
+        System.out.println("extension -list -extensionName hdfs-mirroring -fields status,clusters,tags");
+        Assert.assertEquals(TestContext.executeWithURL(
+                "extension -list -extensionName hdfs-mirroring -fields status,clusters,tags"),
0);
+
+        // validate job list results
+        ExtensionJobList jobs = context.getExtensionJobs("hdfs-mirroring", null, null, null,
null, null);
+        Assert.assertEquals(jobs.numJobs, 2);
+        Assert.assertEquals(jobs.job.get(0).jobName, JOB_NAME_1);
+        Assert.assertEquals(jobs.job.get(1).jobName, JOB_NAME_2);
+
+        // list extension job instances
+        System.out.println("extension -instances -jobName " + JOB_NAME_1 + " -fields status,clusters,tags");
+        Assert.assertEquals(TestContext.executeWithURL(
+                "extension -instances -jobName " + JOB_NAME_1 + " -fields status,clusters,tags"),
0);
+        System.out.println("extension -instances -jobName " + JOB_NAME_2 + " -fields status,clusters,tags");
+        Assert.assertEquals(TestContext.executeWithURL(
+                "extension -instances -jobName " + JOB_NAME_2 + " -fields status,clusters,tags"),
0);
+
+        // validate instance list results
+        ExtensionInstanceList instanceList = context.getExtensionInstances(JOB_NAME_1, START_TIME_1,
endTime, "RUNNING",
+                null, null, null, null, null, null);
+        System.out.println("Validate running instances of extension job " + JOB_NAME_1 +
": \n"
+                + instanceList.toString());
+        Assert.assertEquals(instanceList.numEntities, 1);
+        Assert.assertEquals(instanceList.entitySummary.get(0).instances.length, 1);
+        instanceList = context.getExtensionInstances(JOB_NAME_2, START_TIME_1, endTime, "RUNNING",
+                null, null, null, null, null, null);
+        System.out.println("Validate running instances of extension job " + JOB_NAME_2 +
": \n"
+                + instanceList.toString());
+        Assert.assertEquals(instanceList.numEntities, 1);
+        Assert.assertEquals(instanceList.entitySummary.get(0).instances.length, 1);
+
+        // suspend
+        System.out.println("extension -suspend -jobName " + JOB_NAME_1);
+        Assert.assertEquals(TestContext.executeWithURL("extension -suspend -jobName " + JOB_NAME_1),
0);
+
+        // resume
+        System.out.println("extension -resume -jobName " + JOB_NAME_1);
+        Assert.assertEquals(TestContext.executeWithURL("extension -resume -jobName " + JOB_NAME_1),
0);
+
+        // delete
+        System.out.println("extension -delete -jobName " + JOB_NAME_1);
+        Assert.assertEquals(TestContext.executeWithURL("extension -delete -jobName " + JOB_NAME_1),
0);
+
+        // update
+        filePath = TestContext.overlayParametersOverTemplate(
+                TestContext.getTempFile("property", "txt"),
+                HDFS_MIRRORING_PROPERTY_TEMPLATE,
+                getHDFSMirroringProperty(JOB_NAME_2, START_TIME_2, endTime));
+        System.out.println("extension -update -extensionName hdfs-mirroring -file " + filePath);
+        Assert.assertEquals(TestContext.executeWithURL(
+                "extension -update -extensionName hdfs-mirroring -file " + filePath), 0);
+
+        // validate
+        System.out.println("extension -validate -extensionName hdfs-mirroring -file " + filePath);
+        Assert.assertEquals(TestContext.executeWithURL(
+                "extension -validate -extensionName hdfs-mirroring -file " + filePath), 0);
+
+        // failure case: no file input
+        System.out.println("extension -submitAndSchedule -extensionName hdfs-mirroring");
+        Assert.assertEquals(TestContext.executeWithURL(
+                "extension -submitAndSchedule -extensionName hdfs-mirroring"), -1);
+    }
+
+    private Map<String, String> getHDFSMirroringProperty(String jobName, String start,
String end) {
+        Map<String, String> properties = new HashMap<>();
+        properties.put(ExtensionProperties.JOB_NAME.getName(), jobName);
+        properties.put(ExtensionProperties.CLUSTER_NAME.getName(), CLUSTER_NAME);
+        properties.put(ExtensionProperties.VALIDITY_START.getName(), start);
+        properties.put(ExtensionProperties.VALIDITY_END.getName(), end);
+        properties.put(ExtensionProperties.FREQUENCY.getName(), FREQUENCY);
+        properties.put(HdfsMirroringExtensionProperties.SOURCE_DIR.getName(), SOURCE_DIR);
+        properties.put(HdfsMirroringExtensionProperties.SOURCE_CLUSTER.getName(), CLUSTER_NAME);
+        properties.put(HdfsMirroringExtensionProperties.TARGET_DIR.getName(), TARGET_DIR);
+        properties.put(HdfsMirroringExtensionProperties.TARGET_CLUSTER.getName(), CLUSTER_NAME);
+        return properties;
+    }
+}

http://git-wip-us.apache.org/repos/asf/falcon/blob/7b78c39e/webapp/src/test/java/org/apache/falcon/resource/TestContext.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/falcon/resource/TestContext.java b/webapp/src/test/java/org/apache/falcon/resource/TestContext.java
index 3cf5c18..5412608 100644
--- a/webapp/src/test/java/org/apache/falcon/resource/TestContext.java
+++ b/webapp/src/test/java/org/apache/falcon/resource/TestContext.java
@@ -61,6 +61,7 @@ import javax.net.ssl.SSLContext;
 import javax.net.ssl.SSLSession;
 import javax.net.ssl.TrustManager;
 import javax.servlet.ServletInputStream;
+import javax.ws.rs.HttpMethod;
 import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
 import javax.ws.rs.core.UriBuilder;
@@ -288,6 +289,41 @@ public class TestContext extends AbstractTestContext {
         };
     }
 
+    public ExtensionJobList getExtensionJobs(String extensionName, String fields, String
sortOrder, String offset,
+                                             String resultsPerPage, String doAsUser) {
+        WebResource resource = this.service.path("api/extension/list/" + extensionName);
+        resource = addQueryParam(resource, "doAs", doAsUser);
+        resource = addQueryParam(resource, "fields", fields);
+        resource = addQueryParam(resource, "sortOrder", sortOrder);
+        resource = addQueryParam(resource, "offset", offset);
+        resource = addQueryParam(resource, "numResults", resultsPerPage);
+        ClientResponse response = resource.header("Cookie", AUTH_COOKIE_EQ + authenticationToken)
+                .accept(MediaType.APPLICATION_JSON).type(MediaType.TEXT_XML)
+                .method(HttpMethod.GET, ClientResponse.class);
+        return response.getEntity(ExtensionJobList.class);
+    }
+
+    //SUSPEND CHECKSTYLE CHECK ParameterNumberCheck
+    public ExtensionInstanceList getExtensionInstances(String jobName, String nominalStart,
String nominalEnd,
+                                                       String instanceStatus, String fields,
String orderBy,
+                                                       String sortOrder, String offset, String
resultsPerPage,
+                                                       String doAsUser) {
+        WebResource resource = this.service.path("api/extension/instances/" + jobName);
+        resource = addQueryParam(resource, "start", nominalStart);
+        resource = addQueryParam(resource, "end", nominalEnd);
+        resource = addQueryParam(resource, "instanceStatus", instanceStatus);
+        resource = addQueryParam(resource, "doAs", doAsUser);
+        resource = addQueryParam(resource, "fields", fields);
+        resource = addQueryParam(resource, "orderBy", orderBy);
+        resource = addQueryParam(resource, "sortOrder", sortOrder);
+        resource = addQueryParam(resource, "offset", offset);
+        resource = addQueryParam(resource, "numResults", resultsPerPage);
+        ClientResponse response = resource.header("Cookie", AUTH_COOKIE_EQ + authenticationToken)
+                .accept(MediaType.APPLICATION_JSON).type(MediaType.TEXT_XML)
+                .method(HttpMethod.GET, ClientResponse.class);
+        return response.getEntity(ExtensionInstanceList.class);
+    }
+
     public ClientResponse submitAndSchedule(String template, Map<String, String> overlay,
EntityType entityType)
         throws Exception {
         return submitAndSchedule(template, overlay, entityType, null, "", null);
@@ -448,8 +484,12 @@ public class TestContext extends AbstractTestContext {
 
     public static String overlayParametersOverTemplate(String template,
                                                        Map<String, String> overlay)
throws IOException {
-        File tmpFile = getTempFile();
-        OutputStream out = new FileOutputStream(tmpFile);
+        return overlayParametersOverTemplate(getTempFile(), template, overlay);
+    }
+
+    public static String overlayParametersOverTemplate(File file, String template,
+                                                       Map<String, String> overlay)
throws IOException {
+        OutputStream out = new FileOutputStream(file);
 
         InputStreamReader in;
         InputStream resourceAsStream = TestContext.class.getResourceAsStream(template);
@@ -472,7 +512,7 @@ public class TestContext extends AbstractTestContext {
         }
         reader.close();
         out.close();
-        return tmpFile.getAbsolutePath();
+        return file.getAbsolutePath();
     }
 
     @SuppressWarnings("ResultOfMethodCallIgnored")
@@ -579,4 +619,11 @@ public class TestContext extends AbstractTestContext {
     public static int executeWithURL(String command) throws Exception {
         return new FalconCLI().run((command + " -url " + TestContext.BASE_URL).split("\\s+"));
     }
+
+    private WebResource addQueryParam(WebResource resource, String key, String value) {
+        if (StringUtils.isEmpty(value)) {
+            return resource;
+        }
+        return resource.queryParam(key, value);
+    }
 }

http://git-wip-us.apache.org/repos/asf/falcon/blob/7b78c39e/webapp/src/test/resources/hdfs-mirroring-property-template.txt
----------------------------------------------------------------------
diff --git a/webapp/src/test/resources/hdfs-mirroring-property-template.txt b/webapp/src/test/resources/hdfs-mirroring-property-template.txt
new file mode 100644
index 0000000..c01bc01
--- /dev/null
+++ b/webapp/src/test/resources/hdfs-mirroring-property-template.txt
@@ -0,0 +1,9 @@
+jobName=##jobName##
+jobClusterName=##jobClusterName##
+jobValidityStart=##jobValidityStart##
+jobValidityEnd=##jobValidityEnd##
+jobFrequency=##jobFrequency##
+sourceDir=##sourceDir##
+sourceCluster=##sourceCluster##
+targetDir=##targetDir##
+targetCluster=##targetCluster##

http://git-wip-us.apache.org/repos/asf/falcon/blob/7b78c39e/webapp/src/test/resources/hdfs-mirroring-template.xml
----------------------------------------------------------------------
diff --git a/webapp/src/test/resources/hdfs-mirroring-template.xml b/webapp/src/test/resources/hdfs-mirroring-template.xml
new file mode 100644
index 0000000..6c35c5b
--- /dev/null
+++ b/webapp/src/test/resources/hdfs-mirroring-template.xml
@@ -0,0 +1,45 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+
+<process name="##job.name##" xmlns="uri:falcon:process:0.1">
+    <clusters>
+        <!--  source  -->
+        <cluster name="##job.cluster.name##">
+            <validity end="##job.validity.end##" start="##job.validity.start##"/>
+        </cluster>
+    </clusters>
+
+    <tags/>
+
+    <parallel>1</parallel>
+    <!-- Dir replication needs to run only once to catch up -->
+    <order>LAST_ONLY</order>
+    <frequency>##job.frequency##</frequency>
+    <timezone>##job.timezone##</timezone>
+
+    <properties>
+        <property name="oozie.wf.subworkflow.classpath.inheritance" value="true"/>
+    </properties>
+
+    <workflow name="##job.workflow.name##" engine="##job.workflow.engine##"
+              path="##job.workflow.path##" lib="##job.workflow.lib.path##"/>
+    <retry policy="##job.retry.policy##" delay="##job.retry.delay##" attempts="3"/>
+    <notification type="##job.notification.type##" to="##job.notification.receivers##"/>
+    <ACL/>
+</process>

http://git-wip-us.apache.org/repos/asf/falcon/blob/7b78c39e/webapp/src/test/resources/hdfs-snapshot-mirroring-template.xml
----------------------------------------------------------------------
diff --git a/webapp/src/test/resources/hdfs-snapshot-mirroring-template.xml b/webapp/src/test/resources/hdfs-snapshot-mirroring-template.xml
new file mode 100644
index 0000000..29131da
--- /dev/null
+++ b/webapp/src/test/resources/hdfs-snapshot-mirroring-template.xml
@@ -0,0 +1,45 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+
+<process name="##jobName##" xmlns="uri:falcon:process:0.1">
+    <clusters>
+        <!--  source  -->
+        <cluster name="##jobClusterName##">
+            <validity end="##jobValidityEnd##" start="##jobValidityStart##"/>
+        </cluster>
+    </clusters>
+
+    <tags/>
+
+    <parallel>1</parallel>
+    <!-- Replication needs to run only once to catch up -->
+    <order>LAST_ONLY</order>
+    <frequency>##jobFrequency##</frequency>
+    <timezone>##jobTimezone##</timezone>
+
+    <properties>
+        <property name="oozie.wf.subworkflow.classpath.inheritance" value="true"/>
+    </properties>
+
+    <workflow name="##jobWorkflowName##" engine="##jobWorkflowEngine##"
+              path="##jobWorkflowPath##" lib="##jobWorkflowLibPath##"/>
+    <retry policy="##jobRetryPolicy##" delay="##jobRetryDelay##" attempts="3"/>
+    <notification type="##jobNotificationType##" to="##jobNotificationReceivers##"/>
+    <ACL/>
+</process>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/falcon/blob/7b78c39e/webapp/src/test/resources/hive-mirroring-template.xml
----------------------------------------------------------------------
diff --git a/webapp/src/test/resources/hive-mirroring-template.xml b/webapp/src/test/resources/hive-mirroring-template.xml
new file mode 100644
index 0000000..9f28991
--- /dev/null
+++ b/webapp/src/test/resources/hive-mirroring-template.xml
@@ -0,0 +1,45 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+
+<process name="##name##" xmlns="uri:falcon:process:0.1">
+    <clusters>
+        <!--  source  -->
+        <cluster name="##cluster.name##">
+            <validity end="##cluster.validity.end##" start="##cluster.validity.start##"/>
+        </cluster>
+    </clusters>
+
+    <tags/>
+
+    <parallel>1</parallel>
+    <!-- Replication needs to run only once to catch up -->
+    <order>LAST_ONLY</order>
+    <frequency>##process.frequency##</frequency>
+    <timezone>UTC</timezone>
+
+    <properties>
+        <property name="oozie.wf.subworkflow.classpath.inheritance" value="true"/>
+    </properties>
+
+    <workflow name="##workflow.name##" engine="oozie"
+              path="/apps/data-mirroring/workflows/hive-disaster-recovery-workflow.xml" lib="##workflow.lib.path##"/>
+    <retry policy="##retry.policy##" delay="##retry.delay##" attempts="3"/>
+    <notification type="##notification.type##" to="##notification.receivers##"/>
+    <ACL/>
+</process>

http://git-wip-us.apache.org/repos/asf/falcon/blob/7b78c39e/webapp/src/test/resources/startup.properties
----------------------------------------------------------------------
diff --git a/webapp/src/test/resources/startup.properties b/webapp/src/test/resources/startup.properties
index 58018f1..c7d7c87 100644
--- a/webapp/src/test/resources/startup.properties
+++ b/webapp/src/test/resources/startup.properties
@@ -68,6 +68,7 @@ debug.retry.recorder.path=${user.dir}/target/retry
 debug.libext.feed.retention.paths=${falcon.libext}
 debug.libext.feed.replication.paths=${falcon.libext}
 debug.libext.process.paths=${falcon.libext}
+debug.extension.store.uri=jail://global:00/extension/store
 
 *.falcon.cleanup.service.frequency=minutes(5)
 


Mime
View raw message