apex-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "ASF GitHub Bot (JIRA)" <j...@apache.org>
Subject [jira] [Commented] (APEXMALHAR-2006) Stream API Design
Date Thu, 19 May 2016 03:16:13 GMT

    [ https://issues.apache.org/jira/browse/APEXMALHAR-2006?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15290390#comment-15290390
] 

ASF GitHub Bot commented on APEXMALHAR-2006:
--------------------------------------------

Github user siyuanh commented on a diff in the pull request:

    https://github.com/apache/incubator-apex-malhar/pull/261#discussion_r63817611
  
    --- Diff: stream/src/main/java/org/apache/apex/malhar/stream/api/impl/ApexStreamImpl.java
---
    @@ -0,0 +1,493 @@
    +/**
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements.  See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership.  The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License.  You may obtain a copy of the License at
    + *
    + *   http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing,
    + * software distributed under the License is distributed on an
    + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
    + * KIND, either express or implied.  See the License for the
    + * specific language governing permissions and limitations
    + * under the License.
    + */
    +package org.apache.apex.malhar.stream.api.impl;
    +
    +import java.lang.reflect.Field;
    +import java.lang.reflect.Modifier;
    +import java.util.HashSet;
    +import java.util.LinkedList;
    +import java.util.List;
    +import java.util.Map;
    +import java.util.Set;
    +import java.util.concurrent.Callable;
    +
    +import org.apache.apex.malhar.stream.api.ApexStream;
    +import org.apache.apex.malhar.stream.api.function.Function;
    +import org.apache.apex.malhar.stream.api.operator.FunctionOperator;
    +import org.apache.commons.beanutils.BeanUtils;
    +import org.apache.commons.lang3.tuple.Pair;
    +
    +import com.datatorrent.api.Attribute;
    +import com.datatorrent.api.Context;
    +import com.datatorrent.api.DAG;
    +import com.datatorrent.api.LocalMode;
    +import com.datatorrent.api.Operator;
    +import com.datatorrent.lib.algo.UniqueCounter;
    +import com.datatorrent.lib.io.ConsoleOutputOperator;
    +import com.datatorrent.stram.StramLocalCluster;
    +import com.datatorrent.stram.plan.logical.LogicalPlan;
    +
    +/**
    + * Default stream implementation for ApexStream interface.
    + * It creates the dag(execution plan) from stream api
    + */
    +public class ApexStreamImpl<T> implements ApexStream<T>
    +{
    +
    +  private static Set<Attribute> OPERATOR_ATTRIBUTES;
    +
    +  private static Set<Attribute> DAG_ATTRIBUTES;
    +
    +  private static Set<Attribute> INPUT_ATTRIBUTES;
    +
    +  private static Set<Attribute> OUTPUT_ATTRIBUTES;
    +
    +  static {
    +
    +    OPERATOR_ATTRIBUTES = new HashSet<>();
    +    DAG_ATTRIBUTES = new HashSet<>();
    +    INPUT_ATTRIBUTES = new HashSet<>();
    +    OUTPUT_ATTRIBUTES = new HashSet<>();
    +
    +    try {
    +      for (Field field : Context.OperatorContext.class.getDeclaredFields()) {
    +        if (field.getType() == Attribute.class) {
    +          OPERATOR_ATTRIBUTES.add((Attribute)field.get(Context.OperatorContext.class));
    +        }
    +      }
    +
    +      for (Field field : Context.DAGContext.class.getDeclaredFields()) {
    +        if (field.getType() == Attribute.class) {
    +          DAG_ATTRIBUTES.add((Attribute)field.get(Context.DAGContext.class));
    +        }
    +      }
    +    } catch (IllegalAccessException e) {
    +      //Ignore here
    +    }
    +
    +    INPUT_ATTRIBUTES.add(Context.PortContext.PARTITION_PARALLEL);
    +    INPUT_ATTRIBUTES.add(Context.PortContext.AUTO_RECORD);
    +    INPUT_ATTRIBUTES.add(Context.PortContext.STREAM_CODEC);
    +    INPUT_ATTRIBUTES.add(Context.PortContext.TUPLE_CLASS);
    +
    +
    +    OUTPUT_ATTRIBUTES.add(Context.PortContext.QUEUE_CAPACITY);
    +    OUTPUT_ATTRIBUTES.add(Context.PortContext.BUFFER_MEMORY_MB);
    +    OUTPUT_ATTRIBUTES.add(Context.PortContext.SPIN_MILLIS);
    +    OUTPUT_ATTRIBUTES.add(Context.PortContext.UNIFIER_SINGLE_FINAL);
    +    OUTPUT_ATTRIBUTES.add(Context.PortContext.IS_OUTPUT_UNIFIED);
    +    OUTPUT_ATTRIBUTES.add(Context.PortContext.AUTO_RECORD);
    +    OUTPUT_ATTRIBUTES.add(Context.PortContext.STREAM_CODEC);
    +    OUTPUT_ATTRIBUTES.add(Context.PortContext.TUPLE_CLASS);
    +
    +  }
    +
    +  /**
    +   * The extension point of the stream
    +   *
    +   * @param <T>
    +   */
    +  public static class Brick<T>
    +  {
    +
    +    private Operator.OutputPort<T> lastOutput;
    +
    +    private DagMeta.NodeMeta nodeMeta;
    +
    +    private Pair<Operator.OutputPort, Operator.InputPort> lastStream;
    +
    +    public Operator.OutputPort<T> getLastOutput()
    +    {
    +      return lastOutput;
    +    }
    +
    +    public void setLastOutput(Operator.OutputPort<T> lastOutput)
    +    {
    +      this.lastOutput = lastOutput;
    +    }
    +
    +    public void setLastStream(Pair<Operator.OutputPort, Operator.InputPort> lastStream)
    +    {
    +      this.lastStream = lastStream;
    +    }
    +
    +    public Pair<Operator.OutputPort, Operator.InputPort> getLastStream()
    +    {
    +      return lastStream;
    +    }
    +  }
    +
    +  /**
    +   * Graph behind the stream
    +   */
    +  private DagMeta graph;
    +
    +  private ApexStream<T> delegator;
    +
    +  /**
    +   * Right now the stream only support single extend point
    +   * You can have multiple downstream operators connect to this single extend point though
    +   */
    +  private Brick<T> lastBrick;
    +
    +  public Brick<T> getLastBrick()
    +  {
    +    return lastBrick;
    +  }
    +
    +  public void setLastBrick(Brick<T> lastBrick)
    +  {
    +    this.lastBrick = lastBrick;
    +  }
    +
    +  public ApexStreamImpl()
    +  {
    +    graph = new DagMeta();
    +  }
    +
    +  public ApexStreamImpl(ApexStream<T> apexStream)
    +  {
    +    this.delegator = apexStream;
    +    if (delegator != null && delegator instanceof ApexStreamImpl) {
    +      graph = ((ApexStreamImpl)delegator).graph;
    +      lastBrick = ((ApexStreamImpl<T>)delegator).lastBrick;
    +    }
    +  }
    +
    +  public ApexStreamImpl(DagMeta graph)
    +  {
    +    this(graph, null);
    +  }
    +
    +  public ApexStreamImpl(DagMeta graph, Brick<T> lastBrick)
    +  {
    +    this.graph = graph;
    +    this.lastBrick = lastBrick;
    +  }
    +
    +  @Override
    +  public <O, STREAM extends ApexStream<O>> STREAM map(Function.MapFunction<T,
O> mf)
    +  {
    +    return map(mf.toString(), mf);
    +  }
    +
    +  @Override
    +  @SuppressWarnings("unchecked")
    --- End diff --
    
    You mean put it on the class level?


> Stream API Design
> -----------------
>
>                 Key: APEXMALHAR-2006
>                 URL: https://issues.apache.org/jira/browse/APEXMALHAR-2006
>             Project: Apache Apex Malhar
>          Issue Type: Sub-task
>            Reporter: Siyuan Hua
>            Assignee: Siyuan Hua
>             Fix For: 3.4.0
>
>
> Construct DAG in a similar way as Flink/Spark Streaming



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

Mime
View raw message