apex-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vro...@apache.org
Subject [1/2] apex-core git commit: APEXCORE-593 apex cli get-app-package-info could not retrieve properties defined in properties.xml
Date Fri, 24 Mar 2017 20:00:19 GMT
Repository: apex-core
Updated Branches:
  refs/heads/master 5f95ee0e9 -> b4a4e0517


APEXCORE-593 apex cli get-app-package-info could not retrieve properties defined in properties.xml


Project: http://git-wip-us.apache.org/repos/asf/apex-core/repo
Commit: http://git-wip-us.apache.org/repos/asf/apex-core/commit/82f2761d
Tree: http://git-wip-us.apache.org/repos/asf/apex-core/tree/82f2761d
Diff: http://git-wip-us.apache.org/repos/asf/apex-core/diff/82f2761d

Branch: refs/heads/master
Commit: 82f2761dd5c31968e95d85b51a2103b7ebe2f293
Parents: 16d1bf6
Author: vikram <patilvikram@gmail.com>
Authored: Fri Mar 24 10:58:20 2017 +0530
Committer: vikram <patilvikram@gmail.com>
Committed: Fri Mar 24 12:17:58 2017 +0530

----------------------------------------------------------------------
 .../datatorrent/stram/client/AppPackage.java    |  4 ++
 .../stram/client/AppPackageTest.java            | 71 +++++++++++++++++---
 .../java/com/example/mydtapp/Application.java   |  6 +-
 .../java/com/example/mydtapp/Application2.java  |  2 +-
 .../com/example/mydtapp/StdoutOperator.java     | 12 ++++
 .../java/com/example/mydtapp/TestModule.java    | 57 ++++++++++++++++
 .../com/example/mydtapp/TestModuleOperator.java | 48 +++++++++++++
 .../src/main/resources/META-INF/properties.xml  |  8 +++
 8 files changed, 195 insertions(+), 13 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/apex-core/blob/82f2761d/engine/src/main/java/com/datatorrent/stram/client/AppPackage.java
----------------------------------------------------------------------
diff --git a/engine/src/main/java/com/datatorrent/stram/client/AppPackage.java b/engine/src/main/java/com/datatorrent/stram/client/AppPackage.java
index fd95649..238b646 100644
--- a/engine/src/main/java/com/datatorrent/stram/client/AppPackage.java
+++ b/engine/src/main/java/com/datatorrent/stram/client/AppPackage.java
@@ -380,6 +380,10 @@ public class AppPackage extends JarFile
 
     Configuration config = new Configuration();
 
+    for (Map.Entry<String, PropertyInfo> entry : defaultProperties.entrySet()) {
+      config.set(entry.getKey(), entry.getValue().getValue());
+    }
+
     List<String> absClassPath = new ArrayList<>(classPath);
     for (int i = 0; i < absClassPath.size(); i++) {
       String path = absClassPath.get(i);

http://git-wip-us.apache.org/repos/asf/apex-core/blob/82f2761d/engine/src/test/java/com/datatorrent/stram/client/AppPackageTest.java
----------------------------------------------------------------------
diff --git a/engine/src/test/java/com/datatorrent/stram/client/AppPackageTest.java b/engine/src/test/java/com/datatorrent/stram/client/AppPackageTest.java
index aae3913..20550a9 100644
--- a/engine/src/test/java/com/datatorrent/stram/client/AppPackageTest.java
+++ b/engine/src/test/java/com/datatorrent/stram/client/AppPackageTest.java
@@ -27,6 +27,7 @@ import java.util.Set;
 
 import org.codehaus.jackson.JsonGenerationException;
 import org.codehaus.jackson.map.JsonMappingException;
+import org.codehaus.jettison.json.JSONArray;
 import org.codehaus.jettison.json.JSONException;
 import org.codehaus.jettison.json.JSONObject;
 import org.junit.AfterClass;
@@ -95,22 +96,57 @@ public class AppPackageTest
     Assert.assertEquals(System.getProperty("apex.version", "3.4.0"), json.getString("dtEngineVersion"));
     Assert.assertEquals("lib/*.jar", json.getJSONArray("classPath").getString(0));
 
-    JSONObject application1 = json.getJSONArray("applications").getJSONObject(0);
-    JSONObject application2 = json.getJSONArray("applications").getJSONObject(1);
+    // Test if there are fixed number of applications
+    Assert.assertEquals("Number of applications", 2, json.getJSONArray("applications").length());
 
+    JSONArray applications = json.getJSONArray("applications");
     Map<String, JSONObject> apps = new HashMap<>();
 
-    apps.put(application1.getString("name"), application1);
-    apps.put(application2.getString("name"), application2);
+    for (int i = 0; i < applications.length(); i++) {
+      JSONObject application = applications.getJSONObject(i);
+      apps.put(application.getString("name"), application);
+    }
+
+    // Retrieve applications found
+    JSONObject myFirstApplication = apps.get("MyFirstApplication");
+    JSONObject mySecondApplication = apps.get("MySecondApplication");
+
+    // Testing if expected applications are found
+    Assert.assertNotNull("MyFirstApplication not found", myFirstApplication);
+    Assert.assertNotNull("MySecondApplication not found", mySecondApplication);
+
+    // Tests related to MyFirstApplication start
+    Assert.assertEquals("mydtapp-1.0-SNAPSHOT.jar", myFirstApplication.getString("file"));
+    String errorStackTrace = myFirstApplication.getString("errorStackTrace");
+    Assert.assertEquals("ErrorStackTrace", "null", errorStackTrace);
+
+    JSONObject dag = myFirstApplication.getJSONObject("dag");
+    Assert.assertNotNull("MyFirstApplication DAG does not exist", dag);
+    Assert.assertEquals("Number of streams", 1, dag.getJSONArray("streams").length());
+    Assert.assertEquals("Number of operators", 3, dag.getJSONArray("operators").length());
+
+    JSONArray operatorsJSONArray = dag.getJSONArray("operators");
+    Map<String, JSONObject> operatorMap = new HashMap<>();
 
-    Assert.assertEquals(true, apps.containsKey("MyFirstApplication"));
-    Assert.assertEquals(true, apps.containsKey("MySecondApplication"));
+    for (int i = 0; i < operatorsJSONArray.length(); i++) {
+      JSONObject operator = operatorsJSONArray.getJSONObject(i);
+      operatorMap.put(operator.getString("name"), operator);
+    }
+
+    JSONObject operator = operatorMap.get("rand");
+    Assert.assertNotNull("Input Operator not found", operator);
+    Assert.assertEquals("Input operator class", "com.example.mydtapp.RandomNumberGenerator",
operator.getJSONObject("properties").get("@class"));
 
-    Assert.assertEquals("mydtapp-1.0-SNAPSHOT.jar", apps.get("MyFirstApplication").getString("file"));
+    operator = operatorMap.get("stdout");
+    Assert.assertNotNull("Output Operator not found", operator);
+    Assert.assertEquals("Value for Output Operator's testInput:", "default-value-1", operator.getJSONObject("properties").get("testInput"));
+    Assert.assertEquals("Output Operator class", "com.example.mydtapp.StdoutOperator", operator.getJSONObject("properties").get("@class"));
 
-    JSONObject dag = apps.get("MyFirstApplication").getJSONObject("dag");
-    Assert.assertTrue("There is at least one stream", dag.getJSONArray("streams").length()
>= 1);
-    Assert.assertEquals("There are two operator", 2, dag.getJSONArray("operators").length());
+    operator = operatorMap.get("testModule$testModuleOperator");
+    Assert.assertNotNull("Test Module Operator not found", operator);
+    Assert.assertEquals("Test Module Operator class", "com.example.mydtapp.TestModuleOperator",
operator.getJSONObject("properties").get("@class"));
+    Assert.assertEquals("Value for Test Module Operator's testInput:", "default-value-1",
operator.getJSONObject("properties").get("testInput"));
+    // Tests related to MyFirstApplication end
 
     Assert.assertTrue("app package extraction folder should be retained", new File("target/testapp").exists());
     yap.cleanContent();
@@ -158,4 +194,19 @@ public class AppPackageTest
     result = jomp.getContext(null).writeValueAsString(propertyInfo);
     Assert.assertEquals("{\"value\":\"test-value\",\"description\":\"test-description\"}",
result);
   }
+
+  public void testDefaultProperties()
+  {
+
+    Map<String, AppPackage.PropertyInfo> defaultProperties = ap.getDefaultProperties();
+    Assert.assertEquals(8, defaultProperties.size());
+    Assert.assertEquals("package-default", defaultProperties.get("dt.test.1").getValue());
+    Assert.assertEquals("package-default", defaultProperties.get("dt.test.2").getValue());
+    Assert.assertEquals("package-default", defaultProperties.get("dt.test.3").getValue());
+    Assert.assertEquals("package-default", defaultProperties.get("dt.test.4").getValue());
+    Assert.assertEquals("package-default", defaultProperties.get("dt.test.5").getValue());
+    Assert.assertEquals("package-default", defaultProperties.get("dt.test.6").getValue());
+    Assert.assertEquals("default-value-1", defaultProperties.get("dt.operator.testModule.prop.testInput").getValue());
+    Assert.assertEquals("default-value-1", defaultProperties.get("dt.operator.stdout.prop.testInput").getValue());
+  }
 }

http://git-wip-us.apache.org/repos/asf/apex-core/blob/82f2761d/engine/src/test/resources/testAppPackage/mydtapp/src/main/java/com/example/mydtapp/Application.java
----------------------------------------------------------------------
diff --git a/engine/src/test/resources/testAppPackage/mydtapp/src/main/java/com/example/mydtapp/Application.java
b/engine/src/test/resources/testAppPackage/mydtapp/src/main/java/com/example/mydtapp/Application.java
index 67b2569..1431beb 100644
--- a/engine/src/test/resources/testAppPackage/mydtapp/src/main/java/com/example/mydtapp/Application.java
+++ b/engine/src/test/resources/testAppPackage/mydtapp/src/main/java/com/example/mydtapp/Application.java
@@ -20,12 +20,13 @@ package com.example.mydtapp;
 
 import org.apache.hadoop.conf.Configuration;
 
+import com.datatorrent.api.Module;
 import com.datatorrent.api.annotation.ApplicationAnnotation;
 import com.datatorrent.api.StreamingApplication;
 import com.datatorrent.api.DAG;
 import com.datatorrent.api.DAG.Locality;
 
-@ApplicationAnnotation(name="MyFirstApplication")
+@ApplicationAnnotation(name = "MyFirstApplication")
 public class Application implements StreamingApplication
 {
 
@@ -37,7 +38,8 @@ public class Application implements StreamingApplication
 
     RandomNumberGenerator rand = dag.addOperator("rand", RandomNumberGenerator.class);
     StdoutOperator stdout = dag.addOperator("stdout", new StdoutOperator());
-
+    // This module will be added to dag for testing purpose but will not be connected in
a dag.
+    Module testModule = dag.addModule("testModule", com.example.mydtapp.TestModule.class);
     dag.addStream("data", rand.out, stdout.in).setLocality(Locality.CONTAINER_LOCAL);
   }
 }

http://git-wip-us.apache.org/repos/asf/apex-core/blob/82f2761d/engine/src/test/resources/testAppPackage/mydtapp/src/main/java/com/example/mydtapp/Application2.java
----------------------------------------------------------------------
diff --git a/engine/src/test/resources/testAppPackage/mydtapp/src/main/java/com/example/mydtapp/Application2.java
b/engine/src/test/resources/testAppPackage/mydtapp/src/main/java/com/example/mydtapp/Application2.java
index cc4b6d5..76d4777 100644
--- a/engine/src/test/resources/testAppPackage/mydtapp/src/main/java/com/example/mydtapp/Application2.java
+++ b/engine/src/test/resources/testAppPackage/mydtapp/src/main/java/com/example/mydtapp/Application2.java
@@ -25,7 +25,7 @@ import com.datatorrent.api.StreamingApplication;
 import com.datatorrent.api.DAG;
 import com.datatorrent.api.DAG.Locality;
 
-@ApplicationAnnotation(name="MySecondApplication")
+@ApplicationAnnotation(name = "MySecondApplication")
 public class Application2 implements StreamingApplication
 {
 

http://git-wip-us.apache.org/repos/asf/apex-core/blob/82f2761d/engine/src/test/resources/testAppPackage/mydtapp/src/main/java/com/example/mydtapp/StdoutOperator.java
----------------------------------------------------------------------
diff --git a/engine/src/test/resources/testAppPackage/mydtapp/src/main/java/com/example/mydtapp/StdoutOperator.java
b/engine/src/test/resources/testAppPackage/mydtapp/src/main/java/com/example/mydtapp/StdoutOperator.java
index b4729c9..6e6c4f6 100644
--- a/engine/src/test/resources/testAppPackage/mydtapp/src/main/java/com/example/mydtapp/StdoutOperator.java
+++ b/engine/src/test/resources/testAppPackage/mydtapp/src/main/java/com/example/mydtapp/StdoutOperator.java
@@ -27,6 +27,18 @@ import com.datatorrent.common.util.BaseOperator;
 
 public class StdoutOperator extends BaseOperator
 {
+  private String testInput;
+
+  public String getTestInput()
+  {
+    return testInput;
+  }
+
+  public void setTestInput(String testInput)
+  {
+    this.testInput = testInput;
+  }
+
   public final transient DefaultInputPort<Object> in = new DefaultInputPort<Object>()
   {
     @Override

http://git-wip-us.apache.org/repos/asf/apex-core/blob/82f2761d/engine/src/test/resources/testAppPackage/mydtapp/src/main/java/com/example/mydtapp/TestModule.java
----------------------------------------------------------------------
diff --git a/engine/src/test/resources/testAppPackage/mydtapp/src/main/java/com/example/mydtapp/TestModule.java
b/engine/src/test/resources/testAppPackage/mydtapp/src/main/java/com/example/mydtapp/TestModule.java
new file mode 100644
index 0000000..6aab91a
--- /dev/null
+++ b/engine/src/test/resources/testAppPackage/mydtapp/src/main/java/com/example/mydtapp/TestModule.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.example.mydtapp;
+
+import org.apache.hadoop.conf.Configuration;
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.Module;
+
+/**
+ * This is a test module which can be added to DAG . It will add one dummy operator as well.
+ */
+public class TestModule implements Module
+{
+  private String testInput;
+
+  public final transient ProxyOutputPort<String> outputs = new ProxyOutputPort<String>();
+
+  public final transient ProxyInputPort<String> inputs = new ProxyInputPort<String>();
+
+  @Override
+  public void populateDAG(DAG dag, Configuration configuration)
+  {
+    if (testInput == null) {
+      throw new NullPointerException();
+    }
+
+    TestModuleOperator testModuleOperator = dag.addOperator("testModuleOperator", TestModuleOperator.class);
+    testModuleOperator.setTestInput(this.getTestInput());
+  }
+
+  public String getTestInput()
+  {
+    return testInput;
+  }
+
+  public void setTestInput(String testInput)
+  {
+    this.testInput = testInput;
+  }
+}

http://git-wip-us.apache.org/repos/asf/apex-core/blob/82f2761d/engine/src/test/resources/testAppPackage/mydtapp/src/main/java/com/example/mydtapp/TestModuleOperator.java
----------------------------------------------------------------------
diff --git a/engine/src/test/resources/testAppPackage/mydtapp/src/main/java/com/example/mydtapp/TestModuleOperator.java
b/engine/src/test/resources/testAppPackage/mydtapp/src/main/java/com/example/mydtapp/TestModuleOperator.java
new file mode 100644
index 0000000..bc09e53
--- /dev/null
+++ b/engine/src/test/resources/testAppPackage/mydtapp/src/main/java/com/example/mydtapp/TestModuleOperator.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.example.mydtapp;
+
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.InputOperator;
+import com.datatorrent.common.util.BaseOperator;
+
+
+public class TestModuleOperator extends BaseOperator implements InputOperator
+{
+
+  private String testInput;
+
+  public String getTestInput()
+  {
+    return testInput;
+  }
+
+  public void setTestInput(String testInput)
+  {
+    this.testInput = testInput;
+  }
+
+  public final transient DefaultOutputPort<Object> out = new DefaultOutputPort<Object>();
+
+  @Override
+  public void emitTuples()
+  {
+
+  }
+}

http://git-wip-us.apache.org/repos/asf/apex-core/blob/82f2761d/engine/src/test/resources/testAppPackage/mydtapp/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/engine/src/test/resources/testAppPackage/mydtapp/src/main/resources/META-INF/properties.xml
b/engine/src/test/resources/testAppPackage/mydtapp/src/main/resources/META-INF/properties.xml
index e074fbe..937a803 100644
--- a/engine/src/test/resources/testAppPackage/mydtapp/src/main/resources/META-INF/properties.xml
+++ b/engine/src/test/resources/testAppPackage/mydtapp/src/main/resources/META-INF/properties.xml
@@ -50,4 +50,12 @@
   <property>
     <name>dt.test.required.2</name>
   </property>
+  <property>
+    <name>dt.operator.testModule.prop.testInput</name>
+    <value>default-value-1</value>
+  </property>
+  <property>
+    <name>dt.operator.stdout.prop.testInput</name>
+    <value>default-value-1</value>
+  </property>
 </configuration>


Mime
View raw message