asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dl...@apache.org
Subject [1/3] asterixdb git commit: [ASTERIXDB-2015][IDX] Introduce Secondary Primary Index
Date Tue, 26 Sep 2017 00:23:37 GMT
Repository: asterixdb
Updated Branches:
  refs/heads/master fd9c6020d -> 8351d2531


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/8351d253/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java
index 75e714b..445fbd8 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java
@@ -181,6 +181,23 @@ public class SecondaryBTreeOperationsHelper extends SecondaryTreeIndexOperations
         return index.getKeyFieldNames().size();
     }
 
+    /**
+     *      ======
+     *     |  SK  |             Bloom filter
+     *      ======
+     *      ====== ======
+     *     |  SK  |  PK  |      comparators, type traits
+     *      ====== ======
+     *      ====== ........
+     *     |  SK  | Filter |    field access evaluators
+     *      ====== ........
+     *      ====== ====== ........
+     *     |  SK  |  PK  | Filter |   record fields
+     *      ====== ====== ........
+     *      ====== ========= ........ ........
+     *     |  PK  | Payload |  Meta  | Filter | enforced record
+     *      ====== ========= ........ ........
+     */
     @Override
     @SuppressWarnings("rawtypes")
     protected void setSecondaryRecDescAndComparators() throws AlgebricksException {

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/8351d253/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/utils/DotFormatBuilder.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/utils/DotFormatBuilder.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/utils/DotFormatBuilder.java
new file mode 100644
index 0000000..1bbda27
--- /dev/null
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/utils/DotFormatBuilder.java
@@ -0,0 +1,253 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing,
+ *  software distributed under the License is distributed on an
+ *  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ *  KIND, either express or implied.  See the License for the
+ *  specific language governing permissions and limitations
+ *  under the License.
+ */
+package org.apache.hyracks.algebricks.core.utils;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public class DotFormatBuilder {
+    private final StringBuilder stringBuilder;
+    private final Set<Node> nodes;
+    private final List<Edge> edges;
+
+    public DotFormatBuilder(StringValue graphName) {
+        this.edges = new ArrayList<>();
+        this.nodes = new HashSet<>();
+        this.stringBuilder = new StringBuilder();
+        this.stringBuilder.append("digraph ").append(graphName).append(" {\n").append("rankdir=BT;\n");
+        this.stringBuilder.append("node [style=\"rounded,filled\",shape=box];\n");
+    }
+
+    public String getDotDocument() {
+        // print edges first
+        for (Edge edge : edges) {
+            stringBuilder.append(edge);
+        }
+        // print nodes
+        for (Node node : nodes) {
+            stringBuilder.append(node);
+        }
+        stringBuilder.append("\n}");
+        return stringBuilder.toString();
+    }
+
+    // point of entry method
+    public Node createNode(StringValue nodeId, StringValue nodeLabel) {
+        Node node = new Node(nodeId, nodeLabel);
+        for (Node existingNode : nodes) {
+            if (node.equals(existingNode)) {
+                existingNode.setNodeLabel(nodeLabel);
+                return existingNode;
+            }
+        }
+        nodes.add(node);
+        return node;
+    }
+
+    // point of entry method
+    public Edge createEdge(final Node source, final Node destination) {
+        // sanity checks if any variable is null?
+        if (source == null || destination == null || !nodes.contains(source) || !nodes.contains(destination)) {
+            return null;
+        }
+
+        // append to edges list
+        Edge newEdge = new Edge(source, destination);
+        edges.add(newEdge);
+
+        return newEdge;
+    }
+
+    public class Node {
+        private final StringValue nodeId;
+        private HashMap<String,AttributeValue> attributes = new HashMap<>();
+
+        // no instantiation
+        private Node(StringValue nodeId, StringValue nodeLabel) {
+            this.nodeId = nodeId;
+            setNodeLabel(nodeLabel);
+        }
+
+        public StringValue getNodeId() {
+            return nodeId;
+        }
+
+        public AttributeValue getNodeLabel() {
+            return attributes.get(Attribute.LABEL);
+        }
+
+        public Node setNodeLabel(StringValue nodeLabel) {
+            if (nodeLabel != null) {
+                attributes.put(Attribute.LABEL, nodeLabel);
+            }
+            return this;
+        }
+
+        public Node setFillColor(Color color) {
+            if (color != null) {
+                attributes.put(Attribute.COLOR, color);
+            }
+            return this;
+        }
+
+        @Override
+        public boolean equals(Object other) {
+            if (!(other instanceof Node)) {
+                return false;
+            }
+            Node otherNode = (Node) other;
+
+            return nodeId.getValue().equals(otherNode.nodeId.getValue());
+        }
+
+        @Override
+        public int hashCode() {
+            return nodeId.getValue().hashCode();
+        }
+
+        @Override
+        public String toString() {
+            StringBuilder nodeString = new StringBuilder();
+            nodeString.append(nodeId).append(" [");
+            for (Map.Entry attribute : attributes.entrySet()) {
+                nodeString.append(attribute.getKey()).append("=").append(attribute.getValue()).append(",");
+            }
+            // remove last ","
+            if (nodeString.charAt(nodeString.length() - 1) == ',') {
+                nodeString.deleteCharAt(nodeString.length() - 1);
+            }
+            nodeString.append("];\n");
+
+            return nodeString.toString();
+        }
+    }
+
+    public class Edge {
+        private final Node source;
+        private final Node destination;
+        private final HashMap<String,AttributeValue> attributes = new HashMap<>();
+
+        // no instantiation
+        private Edge(Node source, Node destination) {
+            this.source = source;
+            this.destination = destination;
+        }
+
+        public Edge setLabel(StringValue edgeLabel) {
+            if (edgeLabel != null) {
+                attributes.put(Attribute.LABEL, edgeLabel);
+            }
+            return this;
+        }
+
+        public Edge setColor(Color color) {
+            if (color != null) {
+                attributes.put(Attribute.COLOR, color);
+            }
+            return this;
+        }
+
+        @Override
+        public boolean equals(Object other) {
+            if (!(other instanceof Edge)) {
+                return false;
+            }
+            Edge otherEdge = (Edge) other;
+
+            return source.equals(otherEdge.source) && destination.equals(otherEdge.destination);
+        }
+
+        @Override
+        public int hashCode() {
+            return source.hashCode() ^ destination.hashCode();
+        }
+
+        @Override
+        public String toString() {
+            StringBuilder edgeString = new StringBuilder();
+            edgeString.append(source.getNodeId()).append("->").append(destination.getNodeId()).append(" [");
+            for (Map.Entry attribute : attributes.entrySet()) {
+                edgeString.append(attribute.getKey()).append("=").append(attribute.getValue()).append(",");
+            }
+            // remove last ","
+            if (edgeString.charAt(edgeString.length() - 1) == ',') {
+                edgeString.deleteCharAt(edgeString.length() - 1);
+            }
+            edgeString.append("];\n");
+
+            return edgeString.toString();
+        }
+    }
+
+    public abstract static class AttributeValue {
+        private final String value;
+
+        // no instantiation
+        private AttributeValue(String value) {
+            this.value = value;
+        }
+
+        public String getValue() {
+            return value;
+        }
+
+        @Override
+        public String toString() {
+            return value;
+        }
+    }
+
+    public static final class StringValue extends AttributeValue {
+        // no instantiation
+        private StringValue (String value) {
+            super(value);
+        }
+
+        public static StringValue of(String value) {
+            String newValue = value;
+            if (value == null) {
+                newValue = "";
+            }
+            return new StringValue("\"" + newValue.replace("\"","\'").trim() + "\"");
+        }
+    }
+
+    public static final class Color extends AttributeValue {
+        public static final Color RED = new Color("red");
+        public static final Color SKYBLUE = new Color("skyblue");
+
+        // no instantiation
+        private Color (String color) {
+            super(color);
+        }
+    }
+
+    private class Attribute {
+        private static final String COLOR = "color";
+        private static final String LABEL = "label";
+
+        // no instantiation
+        private Attribute() {
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/8351d253/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/utils/DotFormatGenerator.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/utils/DotFormatGenerator.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/utils/DotFormatGenerator.java
new file mode 100644
index 0000000..392bf44
--- /dev/null
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/utils/DotFormatGenerator.java
@@ -0,0 +1,277 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing,
+ *  software distributed under the License is distributed on an
+ *  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ *  KIND, either express or implied.  See the License for the
+ *  specific language governing permissions and limitations
+ *  under the License.
+ */
+package org.apache.hyracks.algebricks.core.utils;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.tuple.ImmutablePair;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import org.apache.hyracks.algebricks.core.algebra.base.IPhysicalOperator;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractReplicateOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
+import org.apache.hyracks.api.dataflow.ActivityId;
+import org.apache.hyracks.api.dataflow.ConnectorDescriptorId;
+import org.apache.hyracks.api.dataflow.IActivity;
+import org.apache.hyracks.api.dataflow.IConnectorDescriptor;
+import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
+import org.apache.hyracks.api.job.JobActivityGraph;
+import org.apache.hyracks.api.job.JobSpecification;
+
+public class DotFormatGenerator {
+
+    private DotFormatGenerator() {
+    }
+
+    /**
+     * Generates DOT format for {@link JobActivityGraph} that can be visualized
+     * using any DOT format visualizer.
+     *
+     * @param jobActivityGraph The job activity graph
+     * @return DOT format
+     */
+    public static String generate(final JobActivityGraph jobActivityGraph) {
+        final DotFormatBuilder graphBuilder =
+                new DotFormatBuilder(DotFormatBuilder.StringValue.of("JobActivityGraph"));
+        List<IConnectorDescriptor> connectors;
+        IActivity activity;
+        ActivityId fromActivityId;
+        ActivityId toActivityId;
+        String fromFullClassName;
+        String toFullClassName;
+        String fromClassName;
+        String toClassName;
+        DotFormatBuilder.Node fromNode;
+        DotFormatBuilder.Node toNode;
+        final Set<Pair<ActivityId, ActivityId>> activitiesPairedSet = new HashSet<>();
+        final Map<ActivityId, IActivity> activityMap = jobActivityGraph.getActivityMap();
+        final Map<ActivityId, List<IConnectorDescriptor>> activityInputMap = jobActivityGraph.getActivityInputMap();
+        final Map<ActivityId, List<IConnectorDescriptor>> activityOutputMap = jobActivityGraph.getActivityOutputMap();
+
+        // go through each activity. First, map its input -> activity, then activity -> its output
+        for (Map.Entry<ActivityId, IActivity> entry : activityMap.entrySet()) {
+            toFullClassName = entry.getValue().getClass().getName();
+            toClassName = toFullClassName.substring(toFullClassName.lastIndexOf('.') + 1);
+            toActivityId = entry.getValue().getActivityId();
+            toNode = graphBuilder.createNode(DotFormatBuilder.StringValue.of(toActivityId.toString()),
+                    DotFormatBuilder.StringValue.of(toActivityId.toString() + "-" + toClassName));
+            // process input -> to activity
+            connectors = activityInputMap.get(entry.getKey());
+            if (connectors != null) {
+                for (IConnectorDescriptor connector : connectors) {
+                    fromActivityId = jobActivityGraph.getProducerActivity(connector.getConnectorId());
+                    activity = activityMap.get(fromActivityId);
+                    fromFullClassName = activity.getClass().getName();
+                    fromClassName = fromFullClassName.substring(fromFullClassName.lastIndexOf('.') + 1);
+                    fromNode = graphBuilder.createNode(DotFormatBuilder.StringValue.of(fromActivityId.toString()),
+                            DotFormatBuilder.StringValue.of(fromActivityId.toString() + "-" + fromClassName));
+                    Pair<ActivityId, ActivityId> newPair = new ImmutablePair<>(fromActivityId, toActivityId);
+                    if (!activitiesPairedSet.contains(newPair)) {
+                        activitiesPairedSet.add(newPair);
+                        graphBuilder.createEdge(fromNode, toNode);
+                    }
+                }
+            }
+
+            // process from activity -> output
+            fromActivityId = toActivityId;
+            fromNode = toNode;
+            connectors = activityOutputMap.get(entry.getKey());
+            if (connectors != null) {
+                for (IConnectorDescriptor connector : connectors) {
+                    toActivityId = jobActivityGraph.getConsumerActivity(connector.getConnectorId());
+                    activity = activityMap.get(toActivityId);
+                    toFullClassName = activity.getClass().getName();
+                    toClassName = toFullClassName.substring(toFullClassName.lastIndexOf('.') + 1);
+                    toNode = graphBuilder.createNode(DotFormatBuilder.StringValue.of(toActivityId.toString()),
+                            DotFormatBuilder.StringValue.of(toActivityId.toString() + "-" + toClassName));
+                    Pair<ActivityId, ActivityId> newPair = new ImmutablePair<>(fromActivityId, toActivityId);
+                    if (!activitiesPairedSet.contains(newPair)) {
+                        activitiesPairedSet.add(newPair);
+                        graphBuilder.createEdge(fromNode, toNode);
+                    }
+                }
+            }
+        }
+
+        final Map<ActivityId, Set<ActivityId>> blocked2BlockerMap = jobActivityGraph.getBlocked2BlockerMap();
+        IActivity blockedActivity;
+        for (Map.Entry<ActivityId, Set<ActivityId>> entry : blocked2BlockerMap.entrySet()) {
+            blockedActivity = activityMap.get(entry.getKey());
+            toFullClassName = blockedActivity.getClass().getName();
+            toClassName = toFullClassName.substring(toFullClassName.lastIndexOf('.') + 1);
+            toActivityId = entry.getKey();
+            toNode = graphBuilder.createNode(DotFormatBuilder.StringValue.of(toActivityId.toString()),
+                    DotFormatBuilder.StringValue.of(toActivityId.toString() + "-" + toClassName));
+            for (ActivityId blockingActivityId : entry.getValue()) {
+                fromActivityId = blockingActivityId;
+                activity = activityMap.get(fromActivityId);
+                fromFullClassName = activity.getClass().getName();
+                fromClassName = fromFullClassName.substring(fromFullClassName.lastIndexOf('.') + 1);
+                fromNode = graphBuilder.createNode(DotFormatBuilder.StringValue.of(fromActivityId.toString()),
+                        DotFormatBuilder.StringValue.of(fromActivityId.toString() + "-" + fromClassName));
+                Pair<ActivityId, ActivityId> newPair = new ImmutablePair<>(fromActivityId, toActivityId);
+                if (!activitiesPairedSet.contains(newPair)) {
+                    activitiesPairedSet.add(newPair);
+                    graphBuilder.createEdge(fromNode, toNode).setColor(DotFormatBuilder.Color.RED);
+                }
+            }
+        }
+
+        return graphBuilder.getDotDocument();
+    }
+
+    /**
+     * Generates DOT format for {@link JobSpecification} that can be visualized
+     * using any DOT format visualizer.
+     *
+     * @param jobSpecification The job specification
+     * @return DOT format
+     */
+    public static String generate(final JobSpecification jobSpecification) {
+        final DotFormatBuilder graphBuilder =
+                new DotFormatBuilder(DotFormatBuilder.StringValue.of("JobSpecification"));
+        final Map<ConnectorDescriptorId, IConnectorDescriptor> connectorMap = jobSpecification.getConnectorMap();
+        final Map<ConnectorDescriptorId, Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>>>
+                cOp = jobSpecification.getConnectorOperatorMap();
+        ConnectorDescriptorId connectorId;
+        IConnectorDescriptor connector;
+        IOperatorDescriptor leftOperator;
+        IOperatorDescriptor rightOperator;
+        DotFormatBuilder.Node sourceNode;
+        DotFormatBuilder.Node destinationNode;
+        String source;
+        String destination;
+        String edgeLabel;
+        for (Map.Entry<ConnectorDescriptorId,
+                Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>>> entry : cOp.entrySet()) {
+            connectorId = entry.getKey();
+            connector = connectorMap.get(connectorId);
+            edgeLabel = connector.getClass().getName().substring(connector.getClass().getName().lastIndexOf(".") + 1);
+            edgeLabel += "-" + connectorId;
+            leftOperator = entry.getValue().getLeft().getLeft();
+            rightOperator = entry.getValue().getRight().getLeft();
+            source = leftOperator.getClass().getName().substring(
+                    leftOperator.getClass().getName().lastIndexOf(".") + 1);
+            sourceNode = graphBuilder.createNode(DotFormatBuilder.StringValue.of(leftOperator.toString()),
+                    DotFormatBuilder.StringValue.of(leftOperator.toString() + "-" + source));
+            destination = rightOperator.getClass().getName().substring(
+                    rightOperator.getClass().getName().lastIndexOf(".") + 1);
+            destinationNode = graphBuilder.createNode(DotFormatBuilder.StringValue.of(rightOperator.toString()),
+                    DotFormatBuilder.StringValue.of(rightOperator.toString() + "-" + destination));
+            graphBuilder.createEdge(sourceNode, destinationNode).setLabel(DotFormatBuilder.StringValue.of(edgeLabel));
+        }
+
+        return graphBuilder.getDotDocument();
+    }
+
+    /**
+     * Generates DOT format for {@link ILogicalPlan} that can be visualized
+     * using any DOT format visualizer.
+     *
+     * @param plan  The logical plan
+     * @param dotVisitor    The DOT visitor
+     * @return DOT format
+     * @throws AlgebricksException
+     */
+    public static String generate(ILogicalPlan plan, LogicalOperatorDotVisitor dotVisitor) throws AlgebricksException {
+        final DotFormatBuilder graphBuilder = new DotFormatBuilder(DotFormatBuilder.StringValue.of("Plan"));
+        ILogicalOperator root = plan.getRoots().get(0).getValue();
+        generateNode(graphBuilder, root, dotVisitor, new HashSet<>());
+        return graphBuilder.getDotDocument();
+    }
+
+    public static void generateNode(DotFormatBuilder dotBuilder, ILogicalOperator op,
+            LogicalOperatorDotVisitor dotVisitor, Set<ILogicalOperator> operatorsVisited)
+            throws AlgebricksException {
+        DotFormatBuilder.StringValue destinationNodeLabel = formatStringOf(op, dotVisitor);
+        DotFormatBuilder.Node destinationNode = dotBuilder.createNode(DotFormatBuilder.StringValue.of(
+                Integer.toString(op.hashCode())), destinationNodeLabel);
+        DotFormatBuilder.StringValue sourceNodeLabel;
+        DotFormatBuilder.Node sourceNode;
+        for (Mutable<ILogicalOperator> child : op.getInputs()) {
+            sourceNodeLabel = formatStringOf(child.getValue(), dotVisitor);
+            sourceNode = dotBuilder.createNode(DotFormatBuilder.StringValue.of(
+                    Integer.toString(child.getValue().hashCode())), sourceNodeLabel);
+            dotBuilder.createEdge(sourceNode, destinationNode);
+            if (!operatorsVisited.contains(child.getValue())) {
+                generateNode(dotBuilder, child.getValue(), dotVisitor, operatorsVisited);
+            }
+        }
+        if (((AbstractLogicalOperator) op).hasNestedPlans()) {
+            ILogicalOperator nestedOperator;
+            for (ILogicalPlan nestedPlan : ((AbstractOperatorWithNestedPlans) op).getNestedPlans()) {
+                nestedOperator = nestedPlan.getRoots().get(0).getValue();
+                sourceNodeLabel = formatStringOf(nestedOperator, dotVisitor);
+                sourceNode = dotBuilder.createNode(DotFormatBuilder.StringValue.of(
+                        Integer.toString(nestedOperator.hashCode())), sourceNodeLabel);
+                dotBuilder.createEdge(sourceNode, destinationNode).
+                        setLabel(DotFormatBuilder.StringValue.of("subplan"));
+                if (!operatorsVisited.contains(nestedOperator)) {
+                    generateNode(dotBuilder, nestedOperator, dotVisitor, operatorsVisited);
+                }
+            }
+        }
+        if (!(op instanceof ExchangeOperator)) {
+            destinationNode.setFillColor(DotFormatBuilder.Color.SKYBLUE);
+        }
+
+        // replicate/split operator
+        if (op.getOperatorTag() == LogicalOperatorTag.REPLICATE || op.getOperatorTag() == LogicalOperatorTag.SPLIT) {
+            AbstractReplicateOperator replicateOperator = (AbstractReplicateOperator) op;
+            ILogicalOperator replicateOutput;
+            sourceNode = destinationNode;
+            for (int i = 0; i < replicateOperator.getOutputs().size(); i++) {
+                replicateOutput = replicateOperator.getOutputs().get(i).getValue();
+                destinationNodeLabel = formatStringOf(replicateOutput, dotVisitor);
+                destinationNode = dotBuilder.createNode(DotFormatBuilder.StringValue.of(
+                        Integer.toString(replicateOutput.hashCode())), destinationNodeLabel);
+                if (replicateOperator.getOutputMaterializationFlags()[i]) {
+                    dotBuilder.createEdge(sourceNode, destinationNode).setColor(DotFormatBuilder.Color.RED);
+                } else {
+                    dotBuilder.createEdge(sourceNode, destinationNode);
+                }
+            }
+        }
+
+        operatorsVisited.add(op);
+    }
+
+    private static DotFormatBuilder.StringValue formatStringOf(ILogicalOperator operator,
+            LogicalOperatorDotVisitor dotVisitor) throws AlgebricksException {
+        String formattedString = operator.accept(dotVisitor, null).trim();
+        IPhysicalOperator physicalOperator = ((AbstractLogicalOperator)operator).getPhysicalOperator();
+        if (physicalOperator != null) {
+            formattedString += "\\n" + physicalOperator.toString().trim() + " |" + operator.getExecutionMode() + "|";
+        } else {
+            formattedString += "\\n|" + operator.getExecutionMode() + "|";
+        }
+
+        return DotFormatBuilder.StringValue.of(formattedString);
+    }
+}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/8351d253/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/utils/LogicalOperatorDotVisitor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/utils/LogicalOperatorDotVisitor.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/utils/LogicalOperatorDotVisitor.java
new file mode 100644
index 0000000..a54ff63
--- /dev/null
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/utils/LogicalOperatorDotVisitor.java
@@ -0,0 +1,492 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.algebricks.core.utils;
+
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.common.utils.Pair;
+import org.apache.hyracks.algebricks.common.utils.Triple;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractUnnestMapOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.DelegateOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.DistributeResultOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteUpsertOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteUpsertOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteUpsertOperator.Kind;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.IntersectOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterUnnestMapOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterUnnestOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.MaterializeOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.ScriptOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.SplitOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
+import org.apache.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+
+public class LogicalOperatorDotVisitor implements ILogicalOperatorVisitor<String, Void> {
+
+    private final StringBuilder stringBuilder;
+
+    public LogicalOperatorDotVisitor() {
+        stringBuilder = new StringBuilder();
+    }
+
+    @Override
+    public String toString() {
+        return "";
+    }
+
+    private CharSequence str(Object o) {
+        return String.valueOf(o);
+    }
+
+    @Override
+    public String visitAggregateOperator(AggregateOperator op, Void noArgs) throws AlgebricksException {
+        stringBuilder.setLength(0);
+        stringBuilder.append("aggregate ").append(str(op.getVariables())).append(" <- ");
+        pprintExprList(op.getExpressions());
+        return stringBuilder.toString();
+    }
+
+    @Override
+    public String visitRunningAggregateOperator(RunningAggregateOperator op, Void noArgs) throws AlgebricksException {
+        stringBuilder.setLength(0);
+        stringBuilder.append("running-aggregate ").append(str(op.getVariables())).append(" <- ");
+        pprintExprList(op.getExpressions());
+        return stringBuilder.toString();
+    }
+
+    @Override
+    public String visitEmptyTupleSourceOperator(EmptyTupleSourceOperator op, Void noArgs) throws AlgebricksException {
+        stringBuilder.setLength(0);
+        stringBuilder.append("empty-tuple-source");
+        return stringBuilder.toString();
+    }
+
+    @Override
+    public String visitGroupByOperator(GroupByOperator op, Void noArgs) throws AlgebricksException {
+        stringBuilder.setLength(0);
+        stringBuilder.append("group by").append(op.isGroupAll() ? " (all)" : "").append(" (");
+        pprintVeList(op.getGroupByList());
+        stringBuilder.append(") decor (");
+        pprintVeList(op.getDecorList());
+        stringBuilder.append(")");
+        return stringBuilder.toString();
+    }
+
+    @Override
+    public String visitDistinctOperator(DistinctOperator op, Void noArgs) throws AlgebricksException {
+        stringBuilder.setLength(0);
+        stringBuilder.append("distinct (");
+        pprintExprList(op.getExpressions());
+        stringBuilder.append(")");
+        return stringBuilder.toString();
+    }
+
+    @Override
+    public String visitInnerJoinOperator(InnerJoinOperator op, Void noArgs) throws AlgebricksException {
+        stringBuilder.setLength(0);
+        stringBuilder.append("join (").append(op.getCondition().getValue().toString()).append(")");
+        return stringBuilder.toString();
+    }
+
+    @Override
+    public String visitLeftOuterJoinOperator(LeftOuterJoinOperator op, Void noArgs) throws AlgebricksException {
+        stringBuilder.setLength(0);
+        stringBuilder.append("left outer join (").append(op.getCondition().getValue().toString()).append(")");
+        return stringBuilder.toString();
+    }
+
+    @Override
+    public String visitNestedTupleSourceOperator(NestedTupleSourceOperator op, Void noArgs) throws AlgebricksException {
+        stringBuilder.setLength(0);
+        stringBuilder.append("nested tuple source");
+        return stringBuilder.toString();
+    }
+
+    @Override
+    public String visitOrderOperator(OrderOperator op, Void noArgs) throws AlgebricksException {
+        stringBuilder.setLength(0);
+        stringBuilder.append("order ");
+        for (Pair<OrderOperator.IOrder, Mutable<ILogicalExpression>> p : op.getOrderExpressions()) {
+            if (op.getTopK() != -1) {
+                stringBuilder.append("(topK: ").append(op.getTopK()).append(") ");
+            }
+            String fst = getOrderString(p.first);
+            stringBuilder.append("(").append(fst).append(", ").append(p.second.getValue().toString()).append(") ");
+        }
+        return stringBuilder.toString();
+    }
+
+    private String getOrderString(OrderOperator.IOrder first) {
+        switch (first.getKind()) {
+            case ASC:
+                return "ASC";
+            case DESC:
+                return "DESC";
+            default:
+                return first.getExpressionRef().toString();
+        }
+    }
+
+    @Override
+    public String visitAssignOperator(AssignOperator op, Void noArgs) throws AlgebricksException {
+        stringBuilder.setLength(0);
+        stringBuilder.append("assign ").append(str(op.getVariables())).append(" <- ");
+        pprintExprList(op.getExpressions());
+        return stringBuilder.toString();
+    }
+
+    @Override
+    public String visitWriteOperator(WriteOperator op, Void noArgs) throws AlgebricksException {
+        stringBuilder.setLength(0);
+        stringBuilder.append("write ");
+        pprintExprList(op.getExpressions());
+        return stringBuilder.toString();
+    }
+
+    @Override
+    public String visitDistributeResultOperator(DistributeResultOperator op, Void noArgs) throws AlgebricksException {
+        stringBuilder.setLength(0);
+        stringBuilder.append("distribute result ");
+        pprintExprList(op.getExpressions());
+        return stringBuilder.toString();
+    }
+
+    @Override
+    public String visitWriteResultOperator(WriteResultOperator op, Void noArgs) throws AlgebricksException {
+        stringBuilder.setLength(0);
+        stringBuilder.append("load ").append(str(op.getDataSource())).append(" from ")
+                .append(op.getPayloadExpression().getValue().toString()).append(" partitioned by ");
+        pprintExprList(op.getKeyExpressions());
+        return stringBuilder.toString();
+    }
+
+    @Override
+    public String visitSelectOperator(SelectOperator op, Void noArgs) throws AlgebricksException {
+        stringBuilder.setLength(0);
+        stringBuilder.append("select (").append(op.getCondition().getValue().toString()).append(")");
+        return stringBuilder.toString();
+    }
+
+    @Override
+    public String visitProjectOperator(ProjectOperator op, Void noArgs) throws AlgebricksException {
+        stringBuilder.setLength(0);
+        stringBuilder.append("project ").append("(").append(op.getVariables()).append(")");
+        return stringBuilder.toString();
+    }
+
+    @Override
+    public String visitSubplanOperator(SubplanOperator op, Void noArgs) throws AlgebricksException {
+        stringBuilder.setLength(0);
+        stringBuilder.append("subplan {}");
+        return stringBuilder.toString();
+    }
+
+    @Override
+    public String visitUnionOperator(UnionAllOperator op, Void noArgs) throws AlgebricksException {
+        stringBuilder.setLength(0);
+        stringBuilder.append("union");
+        for (Triple<LogicalVariable, LogicalVariable, LogicalVariable> v : op.getVariableMappings()) {
+            stringBuilder.append(" (").append(v.first).append(", ").append(v.second).append(", ").append(v.third)
+                    .append(")");
+        }
+        return stringBuilder.toString();
+    }
+
+    @Override
+    public String visitIntersectOperator(IntersectOperator op, Void noArgs) throws AlgebricksException {
+        stringBuilder.setLength(0);
+        stringBuilder.append("intersect (");
+        stringBuilder.append('[');
+        for (int i = 0; i < op.getOutputVars().size(); i++) {
+            if (i > 0) {
+                stringBuilder.append(", ");
+            }
+            stringBuilder.append(str(op.getOutputVars().get(i)));
+        }
+        stringBuilder.append("] <- [");
+        for (int i = 0; i < op.getNumInput(); i++) {
+            if (i > 0) {
+                stringBuilder.append(", ");
+            }
+            stringBuilder.append('[');
+            for (int j = 0; j < op.getInputVariables(i).size(); j++) {
+                if (j > 0) {
+                    stringBuilder.append(", ");
+                }
+                stringBuilder.append(str(op.getInputVariables(i).get(j)));
+            }
+            stringBuilder.append(']');
+        }
+        stringBuilder.append("])");
+        return stringBuilder.toString();
+    }
+
+    @Override
+    public String visitUnnestOperator(UnnestOperator op, Void noArgs) throws AlgebricksException {
+        stringBuilder.setLength(0);
+        stringBuilder.append("unnest ").append(op.getVariable());
+        if (op.getPositionalVariable() != null) {
+            stringBuilder.append(" at ").append(op.getPositionalVariable());
+        }
+        stringBuilder.append(" <- ").append(op.getExpressionRef().getValue().toString());
+        return stringBuilder.toString();
+    }
+
+    @Override
+    public String visitLeftOuterUnnestOperator(LeftOuterUnnestOperator op, Void noArgs) throws AlgebricksException {
+        stringBuilder.setLength(0);
+        stringBuilder.append("outer-unnest ").append(op.getVariable());
+        if (op.getPositionalVariable() != null) {
+            stringBuilder.append(" at ").append(op.getPositionalVariable());
+        }
+        stringBuilder.append(" <- ").append(op.getExpressionRef().getValue().toString());
+        return stringBuilder.toString();
+    }
+
+    @Override
+    public String visitUnnestMapOperator(UnnestMapOperator op, Void noArgs) throws AlgebricksException {
+        stringBuilder.setLength(0);
+        return printAbstractUnnestMapOperator(op, "unnest-map");
+    }
+
+    @Override
+    public String visitLeftOuterUnnestMapOperator(LeftOuterUnnestMapOperator op, Void noArgs)
+            throws AlgebricksException {
+        stringBuilder.setLength(0);
+        return printAbstractUnnestMapOperator(op, "left-outer-unnest-map");
+    }
+
+    private String printAbstractUnnestMapOperator(AbstractUnnestMapOperator op, String opSignature) {
+        stringBuilder.append(opSignature).append(" ").append(op.getVariables()).append(" <- ")
+                .append(op.getExpressionRef().getValue().toString());
+        appendFilterInformation(stringBuilder, op.getMinFilterVars(), op.getMaxFilterVars());
+        return stringBuilder.toString();
+    }
+
+    @Override
+    public String visitDataScanOperator(DataSourceScanOperator op, Void noArgs) throws AlgebricksException {
+        stringBuilder.setLength(0);
+        stringBuilder.append("data-scan ").append(op.getProjectVariables()).append("<-").append(op.getVariables())
+                .append(" <- ").append(op.getDataSource());
+        appendFilterInformation(stringBuilder, op.getMinFilterVars(), op.getMaxFilterVars());
+        return stringBuilder.toString();
+    }
+
+    private String appendFilterInformation(StringBuilder plan, List<LogicalVariable> minFilterVars,
+            List<LogicalVariable> maxFilterVars) {
+        if (minFilterVars != null || maxFilterVars != null) {
+            plan.append(" with filter on");
+        }
+        if (minFilterVars != null) {
+            plan.append(" min:").append(minFilterVars);
+        }
+        if (maxFilterVars != null) {
+            plan.append(" max:").append(maxFilterVars);
+        }
+        return stringBuilder.toString();
+    }
+
+    @Override
+    public String visitLimitOperator(LimitOperator op, Void noArgs) throws AlgebricksException {
+        stringBuilder.setLength(0);
+        stringBuilder.append("limit ").append(op.getMaxObjects().getValue().toString());
+        ILogicalExpression offset = op.getOffset().getValue();
+        if (offset != null) {
+            stringBuilder.append(", ").append(offset.toString());
+        }
+        return stringBuilder.toString();
+    }
+
+    @Override
+    public String visitExchangeOperator(ExchangeOperator op, Void noArgs) throws AlgebricksException {
+        stringBuilder.setLength(0);
+        stringBuilder.append("exchange");
+        return stringBuilder.toString();
+    }
+
+    @Override
+    public String visitScriptOperator(ScriptOperator op, Void noArgs) throws AlgebricksException {
+        stringBuilder.setLength(0);
+        stringBuilder.append("script (in: ").append(op.getInputVariables()).append(") (out: " )
+                .append(op.getOutputVariables()).append(")");
+        return stringBuilder.toString();
+    }
+
+    @Override
+    public String visitReplicateOperator(ReplicateOperator op, Void noArgs) throws AlgebricksException {
+        stringBuilder.setLength(0);
+        stringBuilder.append("replicate");
+        return stringBuilder.toString();
+    }
+
+    @Override
+    public String visitSplitOperator(SplitOperator op, Void noArgs) throws AlgebricksException {
+        stringBuilder.setLength(0);
+        Mutable<ILogicalExpression> branchingExpression = op.getBranchingExpression();
+        stringBuilder.append("split ").append(branchingExpression.getValue().toString());
+        return stringBuilder.toString();
+    }
+
+    @Override
+    public String visitMaterializeOperator(MaterializeOperator op, Void noArgs) throws AlgebricksException {
+        stringBuilder.setLength(0);
+        stringBuilder.append("materialize");
+        return stringBuilder.toString();
+    }
+
+    @Override
+    public String visitInsertDeleteUpsertOperator(InsertDeleteUpsertOperator op, Void noArgs)
+            throws AlgebricksException {
+        stringBuilder.setLength(0);
+        String header = getIndexOpString(op.getOperation());
+        stringBuilder.append(header).append(str(op.getDataSource())).append(" from record: ")
+                .append(op.getPayloadExpression().getValue().toString());
+        if (op.getAdditionalNonFilteringExpressions() != null) {
+            stringBuilder.append(", meta: ");
+            pprintExprList(op.getAdditionalNonFilteringExpressions());
+        }
+        stringBuilder.append(" partitioned by ");
+        pprintExprList(op.getPrimaryKeyExpressions());
+        if (op.getOperation() == Kind.UPSERT) {
+            stringBuilder.append(" out: ([record-before-upsert:").append(op.getBeforeOpRecordVar());
+            if (op.getBeforeOpAdditionalNonFilteringVars() != null) {
+                stringBuilder.append(", additional-before-upsert: ").append(op.getBeforeOpAdditionalNonFilteringVars());
+            }
+            stringBuilder.append("]) ");
+        }
+        if (op.isBulkload()) {
+            stringBuilder.append(" [bulkload]");
+        }
+        return stringBuilder.toString();
+    }
+
+    @Override
+    public String visitIndexInsertDeleteUpsertOperator(IndexInsertDeleteUpsertOperator op, Void noArgs)
+            throws AlgebricksException {
+        stringBuilder.setLength(0);
+        String header = getIndexOpString(op.getOperation());
+        stringBuilder.append(header).append(op.getIndexName()).append(" on ")
+                .append(str(op.getDataSourceIndex().getDataSource())).append(" from ");
+        if (op.getOperation() == Kind.UPSERT) {
+            stringBuilder.append(" replace:");
+            pprintExprList(op.getPrevSecondaryKeyExprs());
+            stringBuilder.append(" with:");
+            pprintExprList(op.getSecondaryKeyExpressions());
+        } else {
+            pprintExprList(op.getSecondaryKeyExpressions());
+        }
+        if (op.isBulkload()) {
+            stringBuilder.append(" [bulkload]");
+        }
+        return stringBuilder.toString();
+    }
+
+    private String getIndexOpString(Kind opKind) {
+        switch (opKind) {
+            case DELETE:
+                return "delete from ";
+            case INSERT:
+                return "insert into ";
+            case UPSERT:
+                return "upsert into ";
+        }
+        return null;
+    }
+
+    @Override
+    public String visitTokenizeOperator(TokenizeOperator op, Void noArgs) throws AlgebricksException {
+        stringBuilder.setLength(0);
+        stringBuilder.append("tokenize ").append(str(op.getTokenizeVars())).append(" <- ");
+        pprintExprList(op.getSecondaryKeyExpressions());
+        return stringBuilder.toString();
+    }
+
+    @Override
+    public String visitSinkOperator(SinkOperator op, Void noArgs) throws AlgebricksException {
+        stringBuilder.setLength(0);
+        stringBuilder.append("sink");
+        return stringBuilder.toString();
+    }
+
+    @Override
+    public String visitDelegateOperator(DelegateOperator op, Void noArgs) throws AlgebricksException {
+        stringBuilder.setLength(0);
+        stringBuilder.append(op.toString());
+        return stringBuilder.toString();
+    }
+
+    private void pprintExprList(List<Mutable<ILogicalExpression>> expressions) {
+        stringBuilder.append("[");
+        boolean first = true;
+        for (Mutable<ILogicalExpression> exprRef : expressions) {
+            if (first) {
+                first = false;
+            } else {
+                stringBuilder.append(", ");
+            }
+            stringBuilder.append(exprRef.getValue().toString());
+        }
+        stringBuilder.append("]");
+    }
+
+    private void pprintVeList(List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> vePairList) {
+        stringBuilder.append("[");
+        boolean fst = true;
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> ve : vePairList) {
+            if (fst) {
+                fst = false;
+            } else {
+                stringBuilder.append("; ");
+            }
+            if (ve.first != null) {
+                stringBuilder.append(ve.first).append(" := ").append(ve.second);
+            } else {
+                stringBuilder.append(ve.second.getValue().toString());
+            }
+        }
+        stringBuilder.append("]");
+    }
+}


Mime
View raw message