apex-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From t..@apache.org
Subject [05/19] apex-malhar git commit: Add dynamic partition example Update index and revise some README files
Date Mon, 27 Mar 2017 15:19:06 GMT
Add dynamic partition example
Update index and revise some README files


Project: http://git-wip-us.apache.org/repos/asf/apex-malhar/repo
Commit: http://git-wip-us.apache.org/repos/asf/apex-malhar/commit/041af061
Tree: http://git-wip-us.apache.org/repos/asf/apex-malhar/tree/041af061
Diff: http://git-wip-us.apache.org/repos/asf/apex-malhar/diff/041af061

Branch: refs/heads/master
Commit: 041af061bbce5f5d498f658b5b152f1a6c337ce6
Parents: a492e22
Author: Munagala V. Ramanath <ram@datatorrent.com>
Authored: Tue Jun 7 09:27:51 2016 -0700
Committer: Lakshmi Prasanna Velineni <lakshmi@datatorrent.com>
Committed: Sun Mar 26 11:43:48 2017 -0700

----------------------------------------------------------------------
 examples/dynamic-partition/README.md            |   8 +
 .../XmlJavadocCommentsExtractor.xsl             |  44 +++
 examples/dynamic-partition/pom.xml              | 274 +++++++++++++++++++
 .../src/assemble/appPackage.xml                 |  43 +++
 .../src/main/java/com/example/dynamic/App.java  |  23 ++
 .../src/main/java/com/example/dynamic/Gen.java  | 169 ++++++++++++
 .../src/main/resources/META-INF/properties.xml  |  25 ++
 .../src/main/resources/unused-log4j.properties  |   9 +
 .../com/example/dynamic/ApplicationTest.java    |  34 +++
 .../src/test/resources/log4j.properties         |  21 ++
 10 files changed, 650 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/041af061/examples/dynamic-partition/README.md
----------------------------------------------------------------------
diff --git a/examples/dynamic-partition/README.md b/examples/dynamic-partition/README.md
new file mode 100644
index 0000000..f82aaf5
--- /dev/null
+++ b/examples/dynamic-partition/README.md
@@ -0,0 +1,8 @@
+This example illustrates the use of dynamic partitioning of an operator.
+
+It uses an input operator that generates random numbers and outputs them to a
+`DevNull` operator (which, as the name suggests, simply discards them).
+
+The input operator starts out with two partitions; after some tuples have been
+processed, a dynamic repartition is triggered via the `StatsListener` interface
+to increase the number of partitions to four.

http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/041af061/examples/dynamic-partition/XmlJavadocCommentsExtractor.xsl
----------------------------------------------------------------------
diff --git a/examples/dynamic-partition/XmlJavadocCommentsExtractor.xsl b/examples/dynamic-partition/XmlJavadocCommentsExtractor.xsl
new file mode 100644
index 0000000..08075a9
--- /dev/null
+++ b/examples/dynamic-partition/XmlJavadocCommentsExtractor.xsl
@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+            http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+-->
+
+<!--
+    Document   : XmlJavadocCommentsExtractor.xsl
+    Created on : September 16, 2014, 11:30 AM
+    Description:
+        The transformation strips off all information except for comments and tags from xml
javadoc generated by xml-doclet.
+-->
+
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
+  <xsl:output method="xml" standalone="yes"/>
+
+  <!-- copy xml by selecting only the following nodes, attributes and text -->
+  <xsl:template match="node()|text()|@*">
+    <xsl:copy>
+      <xsl:apply-templates select="root|package|class|interface|method|field|type|comment|tag|text()|@name|@qualified|@text"/>
+    </xsl:copy>
+  </xsl:template>
+
+  <!-- Strip off the following paths from the selected xml -->
+  <xsl:template match="//root/package/interface/interface
+                      |//root/package/interface/method/@qualified
+                      |//root/package/class/interface
+                      |//root/package/class/class
+                      |//root/package/class/method/@qualified
+                      |//root/package/class/field/@qualified" />
+
+  <xsl:strip-space elements="*"/>
+</xsl:stylesheet>

http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/041af061/examples/dynamic-partition/pom.xml
----------------------------------------------------------------------
diff --git a/examples/dynamic-partition/pom.xml b/examples/dynamic-partition/pom.xml
new file mode 100644
index 0000000..34e91ee
--- /dev/null
+++ b/examples/dynamic-partition/pom.xml
@@ -0,0 +1,274 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  
+  <groupId>com.example</groupId>
+  <version>1.0-SNAPSHOT</version>
+  <artifactId>dynamic-partition</artifactId>
+  <packaging>jar</packaging>
+
+  <!-- change these to the appropriate values -->
+  <name>Dynamic Partitioning</name>
+  <description>Example showing dynamic partitioning</description>
+
+  <properties>
+    <!-- change this if you desire to use a different version of Apex Core -->
+    <apex.version>3.5.0</apex.version>
+    <apex.apppackage.classpath>lib/*.jar</apex.apppackage.classpath>
+  </properties>
+
+  <build>
+    <plugins>
+       <plugin>
+         <groupId>org.apache.maven.plugins</groupId>
+         <artifactId>maven-eclipse-plugin</artifactId>
+         <version>2.9</version>
+         <configuration>
+           <downloadSources>true</downloadSources>
+         </configuration>
+       </plugin>
+       <plugin>
+         <artifactId>maven-compiler-plugin</artifactId>
+         <version>3.3</version>
+         <configuration>
+           <encoding>UTF-8</encoding>
+           <source>1.7</source>
+           <target>1.7</target>
+           <debug>true</debug>
+           <optimize>false</optimize>
+           <showDeprecation>true</showDeprecation>
+           <showWarnings>true</showWarnings>
+         </configuration>
+       </plugin>
+       <plugin>
+         <artifactId>maven-dependency-plugin</artifactId>
+         <version>2.8</version>
+         <executions>
+           <execution>
+             <id>copy-dependencies</id>
+             <phase>prepare-package</phase>
+             <goals>
+               <goal>copy-dependencies</goal>
+             </goals>
+             <configuration>
+               <outputDirectory>target/deps</outputDirectory>
+               <includeScope>runtime</includeScope>
+             </configuration>
+           </execution>
+         </executions>
+       </plugin>
+
+       <plugin>
+         <artifactId>maven-assembly-plugin</artifactId>
+         <executions>
+           <execution>
+             <id>app-package-assembly</id>
+             <phase>package</phase>
+             <goals>
+               <goal>single</goal>
+             </goals>
+             <configuration>
+               <finalName>${project.artifactId}-${project.version}-apexapp</finalName>
+               <appendAssemblyId>false</appendAssemblyId>
+               <descriptors>
+                 <descriptor>src/assemble/appPackage.xml</descriptor>
+               </descriptors>
+               <archiverConfig>
+                 <defaultDirectoryMode>0755</defaultDirectoryMode>
+               </archiverConfig>                  
+               <archive>
+                 <manifestEntries>
+                   <Class-Path>${apex.apppackage.classpath}</Class-Path>
+                   <DT-Engine-Version>${apex.version}</DT-Engine-Version>
+                   <DT-App-Package-Group-Id>${project.groupId}</DT-App-Package-Group-Id>
+                   <DT-App-Package-Name>${project.artifactId}</DT-App-Package-Name>
+                   <DT-App-Package-Version>${project.version}</DT-App-Package-Version>
+                   <DT-App-Package-Display-Name>${project.name}</DT-App-Package-Display-Name>
+                   <DT-App-Package-Description>${project.description}</DT-App-Package-Description>
+                 </manifestEntries>
+               </archive>
+             </configuration>
+           </execution>
+         </executions>
+       </plugin>
+
+       <plugin>
+         <artifactId>maven-antrun-plugin</artifactId>
+         <version>1.7</version>
+         <executions>
+           <execution>
+             <phase>package</phase>
+             <configuration>
+               <target>
+                 <move file="${project.build.directory}/${project.artifactId}-${project.version}-apexapp.jar"
+                       tofile="${project.build.directory}/${project.artifactId}-${project.version}.apa"
/>
+               </target>
+             </configuration>
+             <goals>
+               <goal>run</goal>
+             </goals>
+           </execution>
+           <execution>
+             <!-- create resource directory for xml javadoc-->
+             <id>createJavadocDirectory</id>
+             <phase>generate-resources</phase>
+             <configuration>
+               <tasks>
+                 <delete dir="${project.build.directory}/generated-resources/xml-javadoc"/>
+                 <mkdir dir="${project.build.directory}/generated-resources/xml-javadoc"/>
+               </tasks>
+             </configuration>
+             <goals>
+               <goal>run</goal>
+             </goals>
+           </execution>
+         </executions>
+       </plugin>
+
+       <plugin>
+         <groupId>org.codehaus.mojo</groupId>
+         <artifactId>build-helper-maven-plugin</artifactId>
+         <version>1.9.1</version>
+         <executions>
+           <execution>
+             <id>attach-artifacts</id>
+             <phase>package</phase>
+             <goals>
+               <goal>attach-artifact</goal>
+             </goals>
+             <configuration>
+               <artifacts>
+                 <artifact>
+                   <file>target/${project.artifactId}-${project.version}.apa</file>
+                   <type>apa</type>
+                 </artifact>
+               </artifacts>
+               <skipAttach>false</skipAttach>
+             </configuration>
+           </execution>
+         </executions>
+       </plugin>
+
+      <!-- generate javdoc -->
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-javadoc-plugin</artifactId>
+        <executions>
+          <!-- generate xml javadoc -->
+          <execution>
+            <id>xml-doclet</id>
+            <phase>generate-resources</phase>
+            <goals>
+              <goal>javadoc</goal>
+            </goals>
+            <configuration>
+              <doclet>com.github.markusbernhardt.xmldoclet.XmlDoclet</doclet>
+              <additionalparam>-d ${project.build.directory}/generated-resources/xml-javadoc
-filename ${project.artifactId}-${project.version}-javadoc.xml</additionalparam>
+              <useStandardDocletOptions>false</useStandardDocletOptions>
+              <docletArtifact>
+                <groupId>com.github.markusbernhardt</groupId>
+                <artifactId>xml-doclet</artifactId>
+                <version>1.0.4</version>
+              </docletArtifact>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+      <!-- Transform xml javadoc to stripped down version containing only class/interface
comments and tags-->
+      <plugin>
+        <groupId>org.codehaus.mojo</groupId>
+        <artifactId>xml-maven-plugin</artifactId>
+        <version>1.0</version>
+        <executions>
+          <execution>
+            <id>transform-xmljavadoc</id>
+            <phase>generate-resources</phase>
+            <goals>
+              <goal>transform</goal>
+            </goals>
+          </execution>
+        </executions>
+        <configuration>
+          <transformationSets>
+            <transformationSet>
+              <dir>${project.build.directory}/generated-resources/xml-javadoc</dir>
+              <includes>
+                <include>${project.artifactId}-${project.version}-javadoc.xml</include>
+              </includes>
+              <stylesheet>XmlJavadocCommentsExtractor.xsl</stylesheet>
+              <outputDir>${project.build.directory}/generated-resources/xml-javadoc</outputDir>
+            </transformationSet>
+          </transformationSets>
+        </configuration>
+      </plugin>
+      <!-- copy xml javadoc to class jar -->
+      <plugin>
+        <artifactId>maven-resources-plugin</artifactId>
+        <version>2.6</version>
+        <executions>
+          <execution>
+            <id>copy-resources</id>
+            <phase>process-resources</phase>
+            <goals>
+              <goal>copy-resources</goal>
+            </goals>
+            <configuration>
+              <outputDirectory>${basedir}/target/classes</outputDirectory>
+              <resources>
+                <resource>
+                  <directory>${project.build.directory}/generated-resources/xml-javadoc</directory>
+                  <includes>
+                    <include>${project.artifactId}-${project.version}-javadoc.xml</include>
+                  </includes>
+                  <filtering>true</filtering>
+                </resource>
+              </resources>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+
+    </plugins>
+
+  </build>
+
+  <dependencies>
+    <!-- add your dependencies here -->
+    <dependency>
+      <groupId>org.apache.apex</groupId>
+      <artifactId>malhar-library</artifactId>
+      <version>3.6.0</version>
+      <!-- 
+           If you know that your application does not need transitive dependencies pulled
in by malhar-library,
+           uncomment the following to reduce the size of your app package.
+      -->
+      <!--    
+      <exclusions>
+        <exclusion>
+          <groupId>*</groupId>
+          <artifactId>*</artifactId>
+        </exclusion>
+      </exclusions>
+      -->
+    </dependency>
+    <dependency>
+      <groupId>org.apache.apex</groupId>
+      <artifactId>apex-common</artifactId>
+      <version>${apex.version}</version>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <version>4.10</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.apex</groupId>
+      <artifactId>apex-engine</artifactId>
+      <version>${apex.version}</version>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+
+</project>

http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/041af061/examples/dynamic-partition/src/assemble/appPackage.xml
----------------------------------------------------------------------
diff --git a/examples/dynamic-partition/src/assemble/appPackage.xml b/examples/dynamic-partition/src/assemble/appPackage.xml
new file mode 100644
index 0000000..7ad071c
--- /dev/null
+++ b/examples/dynamic-partition/src/assemble/appPackage.xml
@@ -0,0 +1,43 @@
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2
http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+  <id>appPackage</id>
+  <formats>
+    <format>jar</format>
+  </formats>
+  <includeBaseDirectory>false</includeBaseDirectory>
+  <fileSets>
+    <fileSet>
+      <directory>${basedir}/target/</directory>
+      <outputDirectory>/app</outputDirectory>
+      <includes>
+        <include>${project.artifactId}-${project.version}.jar</include>
+      </includes>
+    </fileSet>
+    <fileSet>
+      <directory>${basedir}/target/deps</directory>
+      <outputDirectory>/lib</outputDirectory>
+    </fileSet>
+    <fileSet>
+      <directory>${basedir}/src/site/conf</directory>
+      <outputDirectory>/conf</outputDirectory>
+      <includes>
+        <include>*.xml</include>
+      </includes>
+    </fileSet>
+    <fileSet>
+      <directory>${basedir}/src/main/resources/META-INF</directory>
+      <outputDirectory>/META-INF</outputDirectory>
+    </fileSet>
+    <fileSet>
+      <directory>${basedir}/src/main/resources/app</directory>
+      <outputDirectory>/app</outputDirectory>
+    </fileSet>
+    <fileSet>
+      <directory>${basedir}/src/main/resources/resources</directory>
+      <outputDirectory>/resources</outputDirectory>
+    </fileSet>
+  </fileSets>
+
+</assembly>
+

http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/041af061/examples/dynamic-partition/src/main/java/com/example/dynamic/App.java
----------------------------------------------------------------------
diff --git a/examples/dynamic-partition/src/main/java/com/example/dynamic/App.java b/examples/dynamic-partition/src/main/java/com/example/dynamic/App.java
new file mode 100644
index 0000000..9eec263
--- /dev/null
+++ b/examples/dynamic-partition/src/main/java/com/example/dynamic/App.java
@@ -0,0 +1,23 @@
+package com.example.dynamic;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.DAG;
+
+import com.datatorrent.lib.stream.DevNull;
+
+@ApplicationAnnotation(name="Dyn")
+public class App implements StreamingApplication
+{
+
+  @Override
+  public void populateDAG(DAG dag, Configuration conf)
+  {
+    Gen gen         = dag.addOperator("gen",     Gen.class);
+    DevNull devNull = dag.addOperator("devNull", DevNull.class);
+
+    dag.addStream("data", gen.out, devNull.data);
+  }
+}

http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/041af061/examples/dynamic-partition/src/main/java/com/example/dynamic/Gen.java
----------------------------------------------------------------------
diff --git a/examples/dynamic-partition/src/main/java/com/example/dynamic/Gen.java b/examples/dynamic-partition/src/main/java/com/example/dynamic/Gen.java
new file mode 100644
index 0000000..4cccd23
--- /dev/null
+++ b/examples/dynamic-partition/src/main/java/com/example/dynamic/Gen.java
@@ -0,0 +1,169 @@
+package com.example.dynamic;
+
+import com.esotericsoftware.kryo.Kryo;
+import com.esotericsoftware.kryo.io.Input;
+import com.esotericsoftware.kryo.io.Output;
+import com.google.common.collect.Lists;
+import org.apache.commons.io.IOUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.validation.constraints.NotNull;
+import java.io.ByteArrayOutputStream;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+import com.datatorrent.api.DefaultPartition;
+import com.datatorrent.api.Partitioner;
+import com.datatorrent.api.StatsListener;
+
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.InputOperator;
+import com.datatorrent.common.util.BaseOperator;
+
+/**
+ * Operator that dynamically partitions itself after 500 tuples have been emitted
+ */
+public class Gen extends BaseOperator implements InputOperator, Partitioner<Gen>, StatsListener
+{
+  private static final Logger LOG = LoggerFactory.getLogger(Gen.class);
+
+  private static final int MAX_PARTITIONS = 4;    // maximum number of partitions
+
+  private int partitions = 2;                     // initial number of partitions
+
+  @NotNull
+  private int numTuples;                          // number of tuples to emit per window
+
+  private transient int count = 0;
+
+  public final transient DefaultOutputPort<Double> out = new DefaultOutputPort<Double>();
+
+  @Override
+  public void partitioned(Map<Integer, Partition<Gen>> map)
+  {
+    if (partitions != map.size()) {
+      String msg = String.format("partitions = %d, map.size = %d%n", partitions, map.size());
+      throw new RuntimeException(msg);
+    }
+  }
+
+  @Override
+  public void beginWindow(long windowId)
+  {
+    count = 0;
+  }
+
+  @Override
+  public void emitTuples()
+  {
+    if (count < numTuples) {
+      ++count;
+      out.emit(Math.random());
+    }
+  }
+
+  public int getNumTuples()
+  {
+    return numTuples;
+  }
+
+  /**
+   * Sets the number of tuples to be emitted every window.
+   * @param numTuples number of tuples
+   */
+  public void setNumTuples(int numTuples)
+  {
+    this.numTuples = numTuples;
+  }
+
+  @Override
+  public Response processStats(BatchedOperatorStats batchedOperatorStats) {
+
+    final long emittedCount = batchedOperatorStats.getTuplesEmittedPSMA();
+
+    // we only perform a single dynamic repartition
+    Response res = new Response();
+    res.repartitionRequired = false;
+    if (emittedCount > 500 && partitions < MAX_PARTITIONS) {
+      LOG.info("processStats: trying repartition of input operator current {} required {}",
+               partitions, MAX_PARTITIONS);
+      LOG.info("**** operator id = {}, window id = {}, tuplesProcessedPSMA = {}, tuplesEmittedPSMA
= {}",
+              batchedOperatorStats.getOperatorId(),
+              batchedOperatorStats.getCurrentWindowId(),
+              batchedOperatorStats.getTuplesProcessedPSMA(),
+              emittedCount);
+      partitions = MAX_PARTITIONS;
+      res.repartitionRequired = true;
+    }
+
+    return res;
+  }  // processStats
+
+  /**
+   * Clone object by serializing and deserializing using Kryo.
+   * Note this is different from using {@link Kryo#copy(Object)}, which will attempt to also
clone transient fields.
+   *
+   * @param kryo kryo object used to clone objects
+   * @param src src object that copy from
+   * @return cloned object
+   */
+  @SuppressWarnings("unchecked")
+  private static <SRC> SRC cloneObject(Kryo kryo, SRC src)
+  {
+    kryo.setClassLoader(src.getClass().getClassLoader());
+    ByteArrayOutputStream bos = null;
+    Output output;
+    Input input = null;
+    try {
+      bos = new ByteArrayOutputStream();
+      output = new Output(bos);
+      kryo.writeObject(output, src);
+      output.close();
+      input = new Input(bos.toByteArray());
+      return (SRC)kryo.readObject(input, src.getClass());
+    } finally {
+      IOUtils.closeQuietly(input);
+      IOUtils.closeQuietly(bos);
+    }
+  }
+
+  @Override
+  public Collection<Partition<Gen>> definePartitions(
+      Collection<Partition<Gen>> list, PartitioningContext context)
+  {
+    if (partitions < 0) {    // error
+      String msg = String.format("Error: Bad value: partitions = %d%n", partitions);
+      LOG.error(msg);
+      throw new RuntimeException(msg);
+    }
+
+    final int prevCount = list.size();
+    if (1 == prevCount) {    // initial call
+      LOG.info("definePartitions: First call, prevCount = {}, partitions = {}",
+               prevCount, partitions);
+    }
+
+    if (prevCount == partitions) {
+      LOG.info("definePartitions: Nothing to do in definePartitions");
+      return list;    // nothing to do
+    }
+
+    LOG.debug("definePartitions: Repartitioning from {} to {}", prevCount, partitions);
+
+    Kryo kryo = new Kryo();
+
+    // return value: new list of partitions (includes old list)
+    List<Partition<Gen>> newPartitions = Lists.newArrayListWithExpectedSize(partitions);
+
+    for (int i = 0; i < partitions; i++) {
+      Gen oper = cloneObject(kryo, this);
+      newPartitions.add(new DefaultPartition<>(oper));
+    }
+
+    LOG.info("definePartition: returning {} partitions", newPartitions.size());
+    return newPartitions;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/041af061/examples/dynamic-partition/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/examples/dynamic-partition/src/main/resources/META-INF/properties.xml b/examples/dynamic-partition/src/main/resources/META-INF/properties.xml
new file mode 100644
index 0000000..5bbb4b0
--- /dev/null
+++ b/examples/dynamic-partition/src/main/resources/META-INF/properties.xml
@@ -0,0 +1,25 @@
+<?xml version="1.0"?>
+<configuration>
+  <!-- 
+  <property>
+    <name>dt.application.{appName}.operator.{opName}.prop.{propName}</name>
+    <value>some-default-value (if value is not specified, it is required from the user
or custom config when launching)</value>
+  </property>
+  -->
+  <!-- memory assigned to app master
+  <property>
+    <name>dt.attr.MASTER_MEMORY_MB</name>
+    <value>1024</value>
+  </property>
+  -->
+
+ <property>
+   <name>dt.application.*.operator.*.attr.MEMORY_MB</name>
+   <value>200</value>
+ </property> <property>
+   <name>dt.application.Dyn.operator.gen.prop.numTuples</name>
+   <value>1000</value>
+ </property>
+ 
+</configuration>
+

http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/041af061/examples/dynamic-partition/src/main/resources/unused-log4j.properties
----------------------------------------------------------------------
diff --git a/examples/dynamic-partition/src/main/resources/unused-log4j.properties b/examples/dynamic-partition/src/main/resources/unused-log4j.properties
new file mode 100644
index 0000000..a83a30b
--- /dev/null
+++ b/examples/dynamic-partition/src/main/resources/unused-log4j.properties
@@ -0,0 +1,9 @@
+log4j.rootLogger=DEBUG,CONSOLE
+
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+
+log4j.logger.org=info
+#log4j.logger.org.apache.commons.beanutils=warn
+log4j.logger.com.datatorrent=info

http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/041af061/examples/dynamic-partition/src/test/java/com/example/dynamic/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/dynamic-partition/src/test/java/com/example/dynamic/ApplicationTest.java
b/examples/dynamic-partition/src/test/java/com/example/dynamic/ApplicationTest.java
new file mode 100644
index 0000000..788b9d3
--- /dev/null
+++ b/examples/dynamic-partition/src/test/java/com/example/dynamic/ApplicationTest.java
@@ -0,0 +1,34 @@
+package com.example.dynamic;
+
+import java.io.IOException;
+
+import javax.validation.ConstraintViolationException;
+
+import org.junit.Assert;
+
+import org.apache.hadoop.conf.Configuration;
+import org.junit.Test;
+
+import com.datatorrent.api.LocalMode;
+//import com.example.myapexapp.Application;
+
+/**
+ * Test the DAG declaration in local mode.
+ */
+public class ApplicationTest {
+
+  @Test
+  public void testApplication() throws IOException, Exception {
+    try {
+      LocalMode lma = LocalMode.newInstance();
+      Configuration conf = new Configuration(false);
+      conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
+      lma.prepareDAG(new App(), conf);
+      LocalMode.Controller lc = lma.getController();
+      lc.run(10000); // runs for 10 seconds and quits
+    } catch (ConstraintViolationException e) {
+      Assert.fail("constraint violations: " + e.getConstraintViolations());
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/041af061/examples/dynamic-partition/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/examples/dynamic-partition/src/test/resources/log4j.properties b/examples/dynamic-partition/src/test/resources/log4j.properties
new file mode 100644
index 0000000..58d6913
--- /dev/null
+++ b/examples/dynamic-partition/src/test/resources/log4j.properties
@@ -0,0 +1,21 @@
+log4j.rootLogger=DEBUG,CONSOLE
+
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+
+log4j.appender.RFA=org.apache.log4j.RollingFileAppender
+log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
+log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+log4j.appender.RFA.File=/tmp/app.log
+
+# to enable, add SYSLOG to rootLogger
+log4j.appender.SYSLOG=org.apache.log4j.net.SyslogAppender
+log4j.appender.SYSLOG.syslogHost=127.0.0.1
+log4j.appender.SYSLOG.layout=org.apache.log4j.PatternLayout
+log4j.appender.SYSLOG.layout.conversionPattern=${dt.cid} %-5p [%t] %c{2} %x - %m%n
+log4j.appender.SYSLOG.Facility=LOCAL1
+
+log4j.logger.org=info
+#log4j.logger.org.apache.commons.beanutils=warn
+#log4j.logger.com.datatorrent=debug


Mime
View raw message