nemo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From johnya...@apache.org
Subject [incubator-nemo] branch master updated: [NEMO-246] Add javadoc generation to CI (#136)
Date Tue, 30 Oct 2018 08:48:12 GMT
This is an automated email from the ASF dual-hosted git repository.

johnyangk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-nemo.git


The following commit(s) were added to refs/heads/master by this push:
     new 728a79d  [NEMO-246] Add javadoc generation to CI (#136)
728a79d is described below

commit 728a79d6c8af58ea969004f82ee1ead457ba68ce
Author: Jangho Seo <jangho@jangho.io>
AuthorDate: Tue Oct 30 17:48:07 2018 +0900

    [NEMO-246] Add javadoc generation to CI (#136)
    
    JIRA: [NEMO-246: Add javadoc generation to CI](https://issues.apache.org/jira/projects/NEMO/issues/NEMO-246)
    
    **Major changes:**
    - For nemo-frontend-spark, configured doclint to ignore missing symbols because some symbols
are written in Scala.
    - Added mvn clean javadoc:javadoc to .travis.yml
    
    **Minor changes to note:**
    - Fixed some javadoc errors according to the reports from doclint.
    - Added docs/ to .gitignore
    - Added docs/ to RAT ignore list
    
    **Tests for the changes:**
    - N/A
    
    **Other comments:**
    - N/A
    
    Closes #136
---
 .gitignore                                            |  1 +
 .travis.yml                                           |  1 +
 compiler/frontend/spark/pom.xml                       | 19 +++++++++++++++++++
 .../nemo/examples/beam/WriteOneFilePerWindow.java     |  2 +-
 pom.xml                                               |  2 ++
 .../executor/bytetransfer/ClosableBlockingQueue.java  |  1 +
 .../runtime/executor/data/BroadcastManagerWorker.java |  2 +-
 .../nemo/runtime/executor/data/SerializerManager.java |  2 +-
 .../data/partitioner/DataSkewHashPartitioner.java     |  3 ---
 .../executor/data/stores/GlusterFileStore.java        |  3 ---
 .../runtime/executor/data/stores/LocalFileStore.java  |  3 ---
 .../nemo/runtime/executor/task/TaskExecutor.java      |  3 ++-
 .../org/apache/nemo/runtime/master/MetricStore.java   |  2 ++
 .../org/apache/nemo/runtime/master/RuntimeMaster.java |  1 +
 .../runtime/master/resource/ExecutorRepresenter.java  |  6 ++++--
 .../master/resource/ResourceSpecification.java        |  2 +-
 16 files changed, 37 insertions(+), 16 deletions(-)

diff --git a/.gitignore b/.gitignore
index 5def0cd..641eb9b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -11,6 +11,7 @@ REEF_YARN_RUNTIME
 target
 generated
 build
+docs/
 #
 # ----------------------------------------------------------------------
 # Files generated by OutputService during runtime
diff --git a/.travis.yml b/.travis.yml
index b98a183..413b2ac 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -28,6 +28,7 @@ script:
   # the following command line builds the project, runs the tests with coverage and then
execute the SonarCloud analysis
   - if [ "$TRAVIS_PULL_REQUEST" == false ]; then travis_retry mvn clean org.jacoco:jacoco-maven-plugin:prepare-agent
verify sonar:sonar -B -q -ff -Dsurefire.useFile=false -Dorg.slf4j.simpleLogger.defaultLogLevel=info;
fi
   - if [ "$TRAVIS_PULL_REQUEST" != false ]; then travis_retry mvn clean verify -B -q -ff
-Dsurefire.useFile=false -Dorg.slf4j.simpleLogger.defaultLogLevel=info; fi
+  - mvn clean install -DskipTests && mvn javadoc:javadoc
 
 notifications:
   slack:
diff --git a/compiler/frontend/spark/pom.xml b/compiler/frontend/spark/pom.xml
index 93f9f4e..c79a8e1 100644
--- a/compiler/frontend/spark/pom.xml
+++ b/compiler/frontend/spark/pom.xml
@@ -115,6 +115,25 @@ under the License.
                     </execution>
                 </executions>
             </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-javadoc-plugin</artifactId>
+                <version>3.0.0</version>
+                <configuration>
+                    <outputDirectory>docs/apidocs</outputDirectory>
+                    <reportOutputDirectory>docs/apidocs</reportOutputDirectory>
+                    <doclint>-missing</doclint>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>aggregate</id>
+                        <goals>
+                            <goal>aggregate</goal>
+                        </goals>
+                        <phase>site</phase>
+                    </execution>
+                </executions>
+            </plugin>
         </plugins>
     </build>
 </project>
diff --git a/examples/beam/src/main/java/org/apache/nemo/examples/beam/WriteOneFilePerWindow.java
b/examples/beam/src/main/java/org/apache/nemo/examples/beam/WriteOneFilePerWindow.java
index fe5d5f7..07a652e 100644
--- a/examples/beam/src/main/java/org/apache/nemo/examples/beam/WriteOneFilePerWindow.java
+++ b/examples/beam/src/main/java/org/apache/nemo/examples/beam/WriteOneFilePerWindow.java
@@ -60,7 +60,7 @@ public final class WriteOneFilePerWindow extends PTransform<PCollection<String>,
     return input.apply(write);
   }
    /**
-   * A {@link FilenamePolicy} produces a base file name for a write based on metadata about
the data
+   * A {@link FileBasedSink.FilenamePolicy} produces a base file name for a write based on
metadata about the data
    * being written. This always includes the shard number and the total number of shards.
For
    * windowed writes, it also includes the window and pane index (a sequence number assigned
to each
    * trigger firing).
diff --git a/pom.xml b/pom.xml
index 2db16fe..9d7e769 100644
--- a/pom.xml
+++ b/pom.xml
@@ -284,6 +284,8 @@ under the License.
                         <exclude>.editorconfig</exclude>
                         <!-- formatter.xml -->
                         <exclude>formatter.xml</exclude>
+                        <!-- javadoc -->
+                        <exclude>**/docs/**</exclude>
                     </excludes>
                 </configuration>
             </plugin>
diff --git a/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/bytetransfer/ClosableBlockingQueue.java
b/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/bytetransfer/ClosableBlockingQueue.java
index c6a9169..3d992e7 100644
--- a/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/bytetransfer/ClosableBlockingQueue.java
+++ b/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/bytetransfer/ClosableBlockingQueue.java
@@ -80,6 +80,7 @@ public final class ClosableBlockingQueue<T> implements AutoCloseable
{
 
   /**
    * Mark the input end of this queue as closed.
+   * @param throwableToSet a throwable to set as the cause
    */
   public synchronized void closeExceptionally(final Throwable throwableToSet) {
     this.throwable = throwableToSet;
diff --git a/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/data/BroadcastManagerWorker.java
b/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/data/BroadcastManagerWorker.java
index c8a87fc..17a62c5 100644
--- a/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/data/BroadcastManagerWorker.java
+++ b/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/data/BroadcastManagerWorker.java
@@ -115,7 +115,7 @@ public final class BroadcastManagerWorker {
    * (i.e., the variable is expressed as an IREdge, and reside in a executor as a block)
    *
    * @param id of the broadcast variable.
-   * @param inputReader
+   * @param inputReader the {@link InputReader} to register.
    */
   public void registerInputReader(final Serializable id,
                                   final InputReader inputReader) {
diff --git a/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/data/SerializerManager.java
b/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/data/SerializerManager.java
index 6a3c647..dc46ccb 100644
--- a/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/data/SerializerManager.java
+++ b/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/data/SerializerManager.java
@@ -48,7 +48,7 @@ public final class SerializerManager {
 
   /**
    * Register a encoderFactory for runtime edge.
-   * This method regards that compression & decompression property are empty.
+   * This method regards that compression and decompression property are empty.
    *
    * @param runtimeEdgeId  id of the runtime edge.
    * @param encoderFactory the corresponding encoder factory.
diff --git a/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/data/partitioner/DataSkewHashPartitioner.java
b/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/data/partitioner/DataSkewHashPartitioner.java
index cc3eddf..a79defc 100644
--- a/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/data/partitioner/DataSkewHashPartitioner.java
+++ b/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/data/partitioner/DataSkewHashPartitioner.java
@@ -58,9 +58,6 @@ public final class DataSkewHashPartitioner implements Partitioner<Integer>
{
     LOG.info("hashRangeBase {} resulting hashRange {}", hashRangeBase, hashRange);
   }
 
-  /**
-   * @see Partitioner#partition(Object).
-   */
   @Override
   public Integer partition(final Object element) {
     return Math.abs(keyExtractor.extractKey(element).hashCode() % hashRange);
diff --git a/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/data/stores/GlusterFileStore.java
b/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/data/stores/GlusterFileStore.java
index 8fc1bbd..a47afea 100644
--- a/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/data/stores/GlusterFileStore.java
+++ b/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/data/stores/GlusterFileStore.java
@@ -62,9 +62,6 @@ public final class GlusterFileStore extends AbstractBlockStore implements
Remote
     new File(fileDirectory).mkdirs();
   }
 
-  /**
-   * @see BlockStore#createBlock(String).
-   */
   @Override
   public Block createBlock(final String blockId) {
     deleteBlock(blockId);
diff --git a/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/data/stores/LocalFileStore.java
b/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/data/stores/LocalFileStore.java
index 7d9f2ed..62758aa 100644
--- a/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/data/stores/LocalFileStore.java
+++ b/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/data/stores/LocalFileStore.java
@@ -53,9 +53,6 @@ public final class LocalFileStore extends LocalBlockStore {
     new File(fileDirectory).mkdirs();
   }
 
-  /**
-   * @see BlockStore#createBlock(String).
-   */
   @Override
   public Block createBlock(final String blockId) {
     deleteBlock(blockId);
diff --git a/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/task/TaskExecutor.java
b/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/task/TaskExecutor.java
index 9600410..a2da4bc 100644
--- a/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/task/TaskExecutor.java
+++ b/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/task/TaskExecutor.java
@@ -92,7 +92,8 @@ public final class TaskExecutor {
    * @param taskStateManager       State manager for this Task.
    * @param intermediateDataIOFactory    For reading from/writing to data to other tasks.
    * @param broadcastManagerWorker For broadcasts.
-   * @param metricMessageSender    For sending metric with execution stats to Master.
+   * @param metricMessageSender    For sending metric with execution stats to the master.
+   * @param persistentConnectionToMasterMap For sending messages to the master.
    */
   public TaskExecutor(final Task task,
                       final DAG<IRVertex, RuntimeEdge<IRVertex>> irVertexDag,
diff --git a/runtime/master/src/main/java/org/apache/nemo/runtime/master/MetricStore.java
b/runtime/master/src/main/java/org/apache/nemo/runtime/master/MetricStore.java
index 0fd5492..42f22aa 100644
--- a/runtime/master/src/main/java/org/apache/nemo/runtime/master/MetricStore.java
+++ b/runtime/master/src/main/java/org/apache/nemo/runtime/master/MetricStore.java
@@ -150,6 +150,7 @@ public final class MetricStore {
   /**
    * Dumps JSON-serialized string of specific metric.
    * @param metricClass class of metric.
+   * @param <T> type of the metric to dump
    * @return dumped JSON string of all metric.
    * @throws IOException when failed to write json.
    */
@@ -223,6 +224,7 @@ public final class MetricStore {
    * send changed metric data to the frontend client. Also this method is synchronized.
    * @param metricClass class of the metric.
    * @param id id of the metric.
+   * @param <T> type of the metric to broadcast
    */
   public synchronized <T extends Metric> void triggerBroadcast(final Class<T>
metricClass, final String id) {
     final MetricBroadcaster metricBroadcaster = MetricBroadcaster.getInstance();
diff --git a/runtime/master/src/main/java/org/apache/nemo/runtime/master/RuntimeMaster.java
b/runtime/master/src/main/java/org/apache/nemo/runtime/master/RuntimeMaster.java
index 3f1dd3c..d20e854 100644
--- a/runtime/master/src/main/java/org/apache/nemo/runtime/master/RuntimeMaster.java
+++ b/runtime/master/src/main/java/org/apache/nemo/runtime/master/RuntimeMaster.java
@@ -164,6 +164,7 @@ public final class RuntimeMaster {
    *
    * @param plan               to execute
    * @param maxScheduleAttempt the max number of times this plan/sub-part of the plan should
be attempted.
+   * @return pair of {@link PlanStateManager} and {@link ScheduledExecutorService}
    */
   public Pair<PlanStateManager, ScheduledExecutorService> execute(final PhysicalPlan
plan,
                                                                   final int maxScheduleAttempt)
{
diff --git a/runtime/master/src/main/java/org/apache/nemo/runtime/master/resource/ExecutorRepresenter.java
b/runtime/master/src/main/java/org/apache/nemo/runtime/master/resource/ExecutorRepresenter.java
index 351542a..63d98d5 100644
--- a/runtime/master/src/main/java/org/apache/nemo/runtime/master/resource/ExecutorRepresenter.java
+++ b/runtime/master/src/main/java/org/apache/nemo/runtime/master/resource/ExecutorRepresenter.java
@@ -91,6 +91,8 @@ public final class ExecutorRepresenter {
 
   /**
    * Marks all Tasks which were running in this executor as failed.
+   *
+   * @return set of identifiers of tasks that were running in this executor.
    */
   public Set<String> onExecutorFailed() {
     failedTasks.addAll(runningComplyingTasks.values());
@@ -104,7 +106,7 @@ public final class ExecutorRepresenter {
 
   /**
    * Marks the Task as running, and sends scheduling message to the executor.
-   * @param task
+   * @param task the task to run
    */
   public void onTaskScheduled(final Task task) {
     (task.getPropertyValue(ResourceSlotProperty.class).orElse(true)
@@ -136,7 +138,7 @@ public final class ExecutorRepresenter {
 
   /**
    * Marks the specified Task as completed.
-   *
+   * @param taskId id of the completed task
    */
   public void onTaskExecutionComplete(final String taskId) {
     final Task completedTask = removeFromRunningTasks(taskId);
diff --git a/runtime/master/src/main/java/org/apache/nemo/runtime/master/resource/ResourceSpecification.java
b/runtime/master/src/main/java/org/apache/nemo/runtime/master/resource/ResourceSpecification.java
index e320338..58fe251 100644
--- a/runtime/master/src/main/java/org/apache/nemo/runtime/master/resource/ResourceSpecification.java
+++ b/runtime/master/src/main/java/org/apache/nemo/runtime/master/resource/ResourceSpecification.java
@@ -75,7 +75,7 @@ public final class ResourceSpecification {
 
   /**
    * @return -1   if this resource is not poisoned. (for all other normal cases)
-   *         >= 0 the expected time to failure by poison. (for fault-handling tests)
+   *         &gt;= 0 the expected time to failure by poison. (for fault-handling tests)
    */
   public int getPoisonSec() {
     return poisonSec;


Mime
View raw message