lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jbern...@apache.org
Subject [04/11] lucene-solr:branch_6x: SOLR-9009: Adds ability to get an Explanation of a Streaming Expression
Date Mon, 02 May 2016 13:29:35 GMT
SOLR-9009: Adds ability to get an Explanation of a Streaming Expression

Conflicts:
	solr/CHANGES.txt


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/d969de41
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/d969de41
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/d969de41

Branch: refs/heads/branch_6x
Commit: d969de41c1fd5660768c746df48e50b46f6d56fa
Parents: 60a9e38
Author: Dennis Gove <dpgove@gmail.com>
Authored: Mon Apr 18 21:34:36 2016 -0400
Committer: jbernste <jbernste@apache.org>
Committed: Mon May 2 08:36:46 2016 -0400

----------------------------------------------------------------------
 solr/CHANGES.txt                                |   4 +-
 .../org/apache/solr/handler/SQLHandler.java     |  72 +++++
 .../org/apache/solr/handler/StreamHandler.java  |  49 ++++
 .../solr/response/TextResponseWriter.java       |   7 +
 .../client/solrj/io/comp/FieldComparator.java   |  13 +
 .../solrj/io/comp/MultipleFieldComparator.java  |  12 +
 .../solr/client/solrj/io/eq/FieldEqualitor.java |  17 +-
 .../solrj/io/eq/MultipleFieldEqualitor.java     |  12 +
 .../solrj/io/graph/GatherNodesStream.java       |  66 ++++-
 .../solrj/io/graph/ShortestPathStream.java      |  29 +-
 .../client/solrj/io/ops/ConcatOperation.java    |  15 +-
 .../client/solrj/io/ops/DistinctOperation.java  |  33 +--
 .../client/solrj/io/ops/GroupOperation.java     |  42 ++-
 .../client/solrj/io/ops/ReplaceOperation.java   |   6 +
 .../solrj/io/ops/ReplaceWithFieldOperation.java |  12 +
 .../solrj/io/ops/ReplaceWithValueOperation.java |  12 +
 .../client/solrj/io/ops/StreamOperation.java    |   3 -
 .../client/solrj/io/stream/CloudSolrStream.java |  32 +-
 .../solrj/io/stream/ComplementStream.java       |  57 +++-
 .../client/solrj/io/stream/DaemonStream.java    |  57 +++-
 .../client/solrj/io/stream/ExceptionStream.java |  14 +
 .../client/solrj/io/stream/FacetStream.java     |  34 ++-
 .../client/solrj/io/stream/HashJoinStream.java  |  41 ++-
 .../client/solrj/io/stream/IntersectStream.java |  56 +++-
 .../solr/client/solrj/io/stream/JDBCStream.java |  37 +++
 .../solr/client/solrj/io/stream/JoinStream.java |  33 ++-
 .../client/solrj/io/stream/MergeStream.java     |  35 ++-
 .../client/solrj/io/stream/ParallelStream.java  |  41 ++-
 .../client/solrj/io/stream/PushBackStream.java  |   7 +-
 .../client/solrj/io/stream/RandomStream.java    |  26 ++
 .../solr/client/solrj/io/stream/RankStream.java |  43 ++-
 .../client/solrj/io/stream/ReducerStream.java   |  37 ++-
 .../client/solrj/io/stream/RollupStream.java    |  43 ++-
 .../client/solrj/io/stream/SelectStream.java    |  41 ++-
 .../solr/client/solrj/io/stream/SolrStream.java |  18 +-
 .../solr/client/solrj/io/stream/SortStream.java |  36 ++-
 .../client/solrj/io/stream/StatsStream.java     |  27 ++
 .../client/solrj/io/stream/TopicStream.java     |  52 +++-
 .../client/solrj/io/stream/TupleStream.java     |  10 +
 .../client/solrj/io/stream/UniqueStream.java    |  37 ++-
 .../client/solrj/io/stream/UpdateStream.java    |  48 ++-
 .../solrj/io/stream/expr/Explanation.java       | 158 ++++++++++
 .../solrj/io/stream/expr/Expressible.java       |   9 +
 .../solrj/io/stream/expr/StreamExplanation.java |  72 +++++
 .../client/solrj/io/stream/metrics/Metric.java  |  19 ++
 .../solrj/io/stream/RecordCountStream.java      |  36 ++-
 .../StreamExpressionToExplanationTest.java      | 290 +++++++++++++++++++
 47 files changed, 1677 insertions(+), 173 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d969de41/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index c870c51..6200020 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -70,7 +70,7 @@ New Features
 
 * SOLR-8349: Allow sharing of large in memory data structures across cores (Gus Heck, noble)
 
-* SOLR-8913: When using a shared filesystem we should store data dir and tlog dir locations in 
+* SOLR-8913: When using a shared filesystem we should store data dir and tlog dir locations in
   the cluster state. (Mark Miller)
 
 * SOLR-8809: Implement Connection.prepareStatement (Kevin Risden)
@@ -79,6 +79,8 @@ New Features
 
 * SOLR-9041: 'core-admin-read' and 'core-admin-edit' are well known permissions (noble)
 
+* SOLR-9009: Adds ability to get an Explanation of a Streaming Expression (Dennis Gove)
+
 Bug Fixes
 ----------------------
 

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d969de41/solr/core/src/java/org/apache/solr/handler/SQLHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/SQLHandler.java b/solr/core/src/java/org/apache/solr/handler/SQLHandler.java
index 28b033a..aa40046 100644
--- a/solr/core/src/java/org/apache/solr/handler/SQLHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/SQLHandler.java
@@ -49,7 +49,10 @@ import org.apache.solr.client.solrj.io.stream.StreamContext;
 import org.apache.solr.client.solrj.io.stream.TupleStream;
 import org.apache.solr.client.solrj.io.stream.ExceptionStream;
 import org.apache.solr.client.solrj.io.stream.UniqueStream;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExplanation;
 import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
 import org.apache.solr.client.solrj.io.stream.metrics.*;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.cloud.ZkStateReader;
@@ -1289,6 +1292,19 @@ public class SQLHandler extends RequestHandlerBase implements SolrCoreAware , Pe
     public void setStreamContext(StreamContext context) {
       stream.setStreamContext(context);
     }
+    
+    @Override
+    public Explanation toExplanation(StreamFactory factory) throws IOException {
+
+      return new StreamExplanation(getStreamNodeId().toString())
+        .withChildren(new Explanation[]{
+          stream.toExplanation(factory)
+        })
+        .withFunctionName("SQL LIMIT")
+        .withExpression("--non-expressible--")
+        .withImplementingClass(this.getClass().getName())
+        .withExpressionType(ExpressionType.STREAM_DECORATOR);
+    }
 
     public Tuple read() throws IOException {
       ++count;
@@ -1348,6 +1364,19 @@ public class SQLHandler extends RequestHandlerBase implements SolrCoreAware , Pe
       children.add(stream);
       return children;
     }
+    
+    @Override
+    public Explanation toExplanation(StreamFactory factory) throws IOException {
+
+      return new StreamExplanation(getStreamNodeId().toString())
+        .withChildren(new Explanation[]{
+          stream.toExplanation(factory)
+        })
+        .withFunctionName("SQL HAVING")
+        .withExpression("--non-expressible--")
+        .withImplementingClass(this.getClass().getName())
+        .withExpressionType(ExpressionType.STREAM_DECORATOR);
+    }
 
     public void setStreamContext(StreamContext context) {
       stream.setStreamContext(context);
@@ -1385,6 +1414,16 @@ public class SQLHandler extends RequestHandlerBase implements SolrCoreAware , Pe
       this.catalogs = new ArrayList<>();
       this.catalogs.add(this.zkHost);
     }
+    
+    @Override
+    public Explanation toExplanation(StreamFactory factory) throws IOException {
+
+      return new StreamExplanation(getStreamNodeId().toString())
+        .withFunctionName("SQL CATALOG")
+        .withExpression("--non-expressible--")
+        .withImplementingClass(this.getClass().getName())
+        .withExpressionType(ExpressionType.STREAM_DECORATOR);
+    }
 
     public Tuple read() throws IOException {
       Map<String, String> fields = new HashMap<>();
@@ -1425,6 +1464,16 @@ public class SQLHandler extends RequestHandlerBase implements SolrCoreAware , Pe
     public void open() throws IOException {
 
     }
+    
+    @Override
+    public Explanation toExplanation(StreamFactory factory) throws IOException {
+
+      return new StreamExplanation(getStreamNodeId().toString())
+        .withFunctionName("SQL SCHEMA")
+        .withExpression("--non-expressible--")
+        .withImplementingClass(this.getClass().getName())
+        .withExpressionType(ExpressionType.STREAM_DECORATOR);
+    }
 
     public Tuple read() throws IOException {
       Map<String, String> fields = new HashMap<>();
@@ -1470,6 +1519,16 @@ public class SQLHandler extends RequestHandlerBase implements SolrCoreAware , Pe
       }
       Collections.sort(this.tables);
     }
+    
+    @Override
+    public Explanation toExplanation(StreamFactory factory) throws IOException {
+
+      return new StreamExplanation(getStreamNodeId().toString())
+        .withFunctionName("SQL TABLE")
+        .withExpression("--non-expressible--")
+        .withImplementingClass(this.getClass().getName())
+        .withExpressionType(ExpressionType.STREAM_DECORATOR);
+    }
 
     public Tuple read() throws IOException {
       Map<String, String> fields = new HashMap<>();
@@ -1517,6 +1576,19 @@ public class SQLHandler extends RequestHandlerBase implements SolrCoreAware , Pe
     public void open() throws IOException {
       this.stream.open();
     }
+    
+    @Override
+    public Explanation toExplanation(StreamFactory factory) throws IOException {
+
+      return new StreamExplanation(getStreamNodeId().toString())
+        .withChildren(new Explanation[]{
+          stream.toExplanation(factory)
+        })
+        .withFunctionName("SQL METADATA")
+        .withExpression("--non-expressible--")
+        .withImplementingClass(this.getClass().getName())
+        .withExpressionType(ExpressionType.STREAM_DECORATOR);
+    }
 
     // Return a metadata tuple as the first tuple and then pass through to the underlying stream.
     public Tuple read() throws IOException {

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d969de41/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/StreamHandler.java b/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
index c33a7d4..91ee096 100644
--- a/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
@@ -35,8 +35,11 @@ import org.apache.solr.client.solrj.io.ops.DistinctOperation;
 import org.apache.solr.client.solrj.io.ops.GroupOperation;
 import org.apache.solr.client.solrj.io.ops.ReplaceOperation;
 import org.apache.solr.client.solrj.io.stream.*;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation;
 import org.apache.solr.client.solrj.io.stream.expr.Expressible;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExplanation;
 import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
 import org.apache.solr.client.solrj.io.stream.metrics.CountMetric;
 import org.apache.solr.client.solrj.io.stream.metrics.MaxMetric;
 import org.apache.solr.client.solrj.io.stream.metrics.MeanMetric;
@@ -192,6 +195,12 @@ public class StreamHandler extends RequestHandlerBase implements SolrCoreAware,
     context.setSolrClientCache(clientCache);
     context.put("core", this.coreName);
     tupleStream.setStreamContext(context);
+    
+    // if asking for explanation then go get it
+    if(params.getBool("explain", false)){
+      rsp.add("explanation", tupleStream.toExplanation(this.streamFactory));
+    }
+    
     if(tupleStream instanceof DaemonStream) {
       DaemonStream daemonStream = (DaemonStream)tupleStream;
       if(daemons.containsKey(daemonStream.getId())) {
@@ -272,6 +281,16 @@ public class StreamHandler extends RequestHandlerBase implements SolrCoreAware,
     public List<TupleStream> children() {
       return null;
     }
+    
+    @Override
+    public Explanation toExplanation(StreamFactory factory) throws IOException {
+
+      return new StreamExplanation(getStreamNodeId().toString())
+        .withFunctionName("error")
+        .withImplementingClass(this.getClass().getName())
+        .withExpressionType(ExpressionType.STREAM_DECORATOR)
+        .withExpression("--non-expressible--");
+    }
 
     public Tuple read() {
       String msg = e.getMessage();
@@ -313,6 +332,16 @@ public class StreamHandler extends RequestHandlerBase implements SolrCoreAware,
       return null;
     }
 
+    @Override
+    public Explanation toExplanation(StreamFactory factory) throws IOException {
+
+      return new StreamExplanation(getStreamNodeId().toString())
+        .withFunctionName("daemon-collection")
+        .withImplementingClass(this.getClass().getName())
+        .withExpressionType(ExpressionType.STREAM_DECORATOR)
+        .withExpression("--non-expressible--");
+    }
+    
     public Tuple read() {
       if(it.hasNext()) {
         return it.next().getInfo();
@@ -348,6 +377,16 @@ public class StreamHandler extends RequestHandlerBase implements SolrCoreAware,
       return null;
     }
 
+    @Override
+    public Explanation toExplanation(StreamFactory factory) throws IOException {
+
+      return new StreamExplanation(getStreamNodeId().toString())
+        .withFunctionName("daemon-response")
+        .withImplementingClass(this.getClass().getName())
+        .withExpressionType(ExpressionType.STREAM_DECORATOR)
+        .withExpression("--non-expressible--");
+    }
+
     public Tuple read() {
       if (sendEOF) {
         Map m = new HashMap();
@@ -392,6 +431,16 @@ public class StreamHandler extends RequestHandlerBase implements SolrCoreAware,
       return this.tupleStream.children();
     }
 
+    @Override
+    public Explanation toExplanation(StreamFactory factory) throws IOException {
+
+      return new StreamExplanation(getStreamNodeId().toString())
+        .withFunctionName("timer")
+        .withImplementingClass(this.getClass().getName())
+        .withExpressionType(ExpressionType.STREAM_DECORATOR)
+        .withExpression("--non-expressible--");
+    }
+
     public Tuple read() throws IOException {
       Tuple tuple = this.tupleStream.read();
       if(tuple.EOF) {

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d969de41/solr/core/src/java/org/apache/solr/response/TextResponseWriter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/response/TextResponseWriter.java b/solr/core/src/java/org/apache/solr/response/TextResponseWriter.java
index bde5759..e1b1dee 100644
--- a/solr/core/src/java/org/apache/solr/response/TextResponseWriter.java
+++ b/solr/core/src/java/org/apache/solr/response/TextResponseWriter.java
@@ -30,6 +30,7 @@ import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.util.BytesRef;
 import org.apache.solr.client.solrj.io.Tuple;
 import org.apache.solr.client.solrj.io.stream.TupleStream;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation;
 import org.apache.solr.common.EnumFieldValue;
 import org.apache.solr.common.SolrDocument;
 import org.apache.solr.common.SolrDocumentList;
@@ -165,6 +166,8 @@ public abstract class TextResponseWriter {
       writeNamedList(name, (NamedList)val);
     } else if (val instanceof TupleStream) {
       writeTupleStream((TupleStream) val);
+    } else if (val instanceof Explanation){
+      writeExplanation((Explanation) val);
     } else if (val instanceof Path) {
       writeStr(name, ((Path) val).toAbsolutePath().toString(), true);
     } else if (val instanceof Iterable) {
@@ -316,6 +319,10 @@ public abstract class TextResponseWriter {
     tupleStream.writeStreamClose(writer);
     tupleStream.close();
   }
+  
+  public void writeExplanation(Explanation explanation) throws IOException {
+    writeMap(null, explanation.toMap(), false, true);
+  }
 
 
   /** if this form of the method is called, val is the Java string form of a double */

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d969de41/solr/solrj/src/java/org/apache/solr/client/solrj/io/comp/FieldComparator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/comp/FieldComparator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/comp/FieldComparator.java
index 10c7c29..15af57a 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/comp/FieldComparator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/comp/FieldComparator.java
@@ -16,9 +16,13 @@
  */
 package org.apache.solr.client.solrj.io.comp;
 
+import java.io.IOException;
 import java.util.Map;
+import java.util.UUID;
 
 import org.apache.solr.client.solrj.io.Tuple;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionValue;
 import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
@@ -29,6 +33,7 @@ import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
 public class FieldComparator implements StreamComparator {
 
   private static final long serialVersionUID = 1;
+  private UUID comparatorNodeId = UUID.randomUUID();
   
   private String leftFieldName;
   private String rightFieldName;
@@ -84,6 +89,14 @@ public class FieldComparator implements StreamComparator {
     
     return new StreamExpressionValue(sb.toString());
   }
+
+  @Override
+  public Explanation toExplanation(StreamFactory factory) throws IOException {
+    return new Explanation(comparatorNodeId.toString())
+      .withExpressionType(ExpressionType.SORTER)
+      .withImplementingClass(getClass().getName())
+      .withExpression(toExpression(factory).toString());
+  }
   
   /*
    * What're we doing here messing around with lambdas for the comparator logic?

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d969de41/solr/solrj/src/java/org/apache/solr/client/solrj/io/comp/MultipleFieldComparator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/comp/MultipleFieldComparator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/comp/MultipleFieldComparator.java
index 44edc25..1370540 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/comp/MultipleFieldComparator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/comp/MultipleFieldComparator.java
@@ -19,12 +19,15 @@ package org.apache.solr.client.solrj.io.comp;
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;
+import java.util.UUID;
 
 import org.apache.solr.client.solrj.io.Tuple;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation;
 import org.apache.solr.client.solrj.io.stream.expr.Expressible;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionValue;
 import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
 
 
 /**
@@ -34,6 +37,7 @@ import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
 public class MultipleFieldComparator implements StreamComparator {
 
   private static final long serialVersionUID = 1;
+  private UUID comparatorNodeId = UUID.randomUUID();
 
   private StreamComparator[] comps;
 
@@ -91,6 +95,14 @@ public class MultipleFieldComparator implements StreamComparator {
     
     return false;
   }
+
+  @Override
+  public Explanation toExplanation(StreamFactory factory) throws IOException {
+    return new Explanation(comparatorNodeId.toString())
+      .withExpressionType(ExpressionType.SORTER)
+      .withImplementingClass(getClass().getName())
+      .withExpression(toExpression(factory).toString());
+  }
   
   @Override
   public MultipleFieldComparator copyAliased(Map<String,String> aliases){

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d969de41/solr/solrj/src/java/org/apache/solr/client/solrj/io/eq/FieldEqualitor.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eq/FieldEqualitor.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eq/FieldEqualitor.java
index 50b8e22..e7e207e 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eq/FieldEqualitor.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eq/FieldEqualitor.java
@@ -16,15 +16,15 @@
  */
 package org.apache.solr.client.solrj.io.eq;
 
-import java.io.Serializable;
-import java.util.Comparator;
+import java.io.IOException;
+import java.util.UUID;
 
 import org.apache.solr.client.solrj.io.Tuple;
-import org.apache.solr.client.solrj.io.comp.ComparatorOrder;
 import org.apache.solr.client.solrj.io.comp.FieldComparator;
 import org.apache.solr.client.solrj.io.comp.MultipleFieldComparator;
 import org.apache.solr.client.solrj.io.comp.StreamComparator;
-import org.apache.solr.client.solrj.io.stream.expr.Expressible;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionValue;
 import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
@@ -35,6 +35,7 @@ import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
 public class FieldEqualitor implements StreamEqualitor {
 
   private static final long serialVersionUID = 1;
+  private UUID equalitorNodeId = UUID.randomUUID();
   
   private String leftFieldName;
   private String rightFieldName;
@@ -63,6 +64,14 @@ public class FieldEqualitor implements StreamEqualitor {
     
     return new StreamExpressionValue(sb.toString());
   }
+
+  @Override
+  public Explanation toExplanation(StreamFactory factory) throws IOException {
+    return new Explanation(equalitorNodeId.toString())
+      .withExpressionType(ExpressionType.EQUALITOR)
+      .withImplementingClass(getClass().getName())
+      .withExpression(toExpression(factory).toString());
+  }
   
   public boolean test(Tuple leftTuple, Tuple rightTuple) {
 

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d969de41/solr/solrj/src/java/org/apache/solr/client/solrj/io/eq/MultipleFieldEqualitor.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eq/MultipleFieldEqualitor.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eq/MultipleFieldEqualitor.java
index 982018e..76c261a 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eq/MultipleFieldEqualitor.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eq/MultipleFieldEqualitor.java
@@ -17,14 +17,17 @@
 package org.apache.solr.client.solrj.io.eq;
 
 import java.io.IOException;
+import java.util.UUID;
 
 import org.apache.solr.client.solrj.io.Tuple;
 import org.apache.solr.client.solrj.io.comp.MultipleFieldComparator;
 import org.apache.solr.client.solrj.io.comp.StreamComparator;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation;
 import org.apache.solr.client.solrj.io.stream.expr.Expressible;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionValue;
 import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
 
 
 /**
@@ -34,6 +37,7 @@ import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
 public class MultipleFieldEqualitor implements StreamEqualitor {
 
   private static final long serialVersionUID = 1;
+  private UUID equalitorNodeId = UUID.randomUUID();
 
   private StreamEqualitor[] eqs;
 
@@ -60,6 +64,14 @@ public class MultipleFieldEqualitor implements StreamEqualitor {
     
     return new StreamExpressionValue(sb.toString());
   }
+
+  @Override
+  public Explanation toExplanation(StreamFactory factory) throws IOException {
+    return new Explanation(equalitorNodeId.toString())
+      .withExpressionType(ExpressionType.EQUALITOR)
+      .withImplementingClass(getClass().getName())
+      .withExpression(toExpression(factory).toString());
+  }
   
   @Override
   public boolean isDerivedFrom(StreamEqualitor base){

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d969de41/solr/solrj/src/java/org/apache/solr/client/solrj/io/graph/GatherNodesStream.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/graph/GatherNodesStream.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/graph/GatherNodesStream.java
index 759aa0f..7ab6e97 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/graph/GatherNodesStream.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/graph/GatherNodesStream.java
@@ -29,6 +29,7 @@ import java.util.ArrayList;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
+import java.util.stream.Collectors;
 
 import org.apache.solr.client.solrj.io.eq.MultipleFieldEqualitor;
 import org.apache.solr.client.solrj.io.stream.*;
@@ -36,12 +37,15 @@ import org.apache.solr.client.solrj.io.stream.metrics.*;
 import org.apache.solr.client.solrj.io.Tuple;
 import org.apache.solr.client.solrj.io.comp.StreamComparator;
 import org.apache.solr.client.solrj.io.eq.FieldEqualitor;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation;
 import org.apache.solr.client.solrj.io.stream.expr.Expressible;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExplanation;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionNamedParameter;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionValue;
 import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
 import org.apache.solr.common.util.ExecutorUtil;
 import org.apache.solr.common.util.SolrjNamedThreadFactory;
 
@@ -50,7 +54,7 @@ public class GatherNodesStream extends TupleStream implements Expressible {
   private String zkHost;
   private String collection;
   private StreamContext streamContext;
-  private Map queryParams;
+  private Map<String,String> queryParams;
   private String traverseFrom;
   private String traverseTo;
   private String gather;
@@ -250,21 +254,30 @@ public class GatherNodesStream extends TupleStream implements Expressible {
   }
 
   @Override
-  public StreamExpressionParameter toExpression(StreamFactory factory) throws IOException {
+  public StreamExpression toExpression(StreamFactory factory) throws IOException{
+    return toExpression(factory, true);
+  }
+  
+  private StreamExpression toExpression(StreamFactory factory, boolean includeStreams) throws IOException {
 
     StreamExpression expression = new StreamExpression(factory.getFunctionName(this.getClass()));
 
     // collection
     expression.addParameter(collection);
 
-    if(tupleStream instanceof Expressible){
-      expression.addParameter(((Expressible)tupleStream).toExpression(factory));
+    if(includeStreams){
+      if(tupleStream instanceof Expressible){
+        expression.addParameter(((Expressible)tupleStream).toExpression(factory));
+      }
+      else{
+        throw new IOException("This GatherNodesStream contains a non-expressible TupleStream - it cannot be converted to an expression");
+      }
     }
     else{
-      throw new IOException("This GatherNodesStream contains a non-expressible TupleStream - it cannot be converted to an expression");
+      expression.addParameter("<stream>");
     }
 
-    Set<Map.Entry> entries =  queryParams.entrySet();
+    Set<Map.Entry<String,String>> entries =  queryParams.entrySet();
     // parameters
     for(Map.Entry param : entries){
       String value = param.getValue().toString();
@@ -300,6 +313,37 @@ public class GatherNodesStream extends TupleStream implements Expressible {
 
     return expression;
   }
+  
+  @Override
+  public Explanation toExplanation(StreamFactory factory) throws IOException {
+
+    StreamExplanation explanation = new StreamExplanation(getStreamNodeId().toString());
+    
+    explanation.setFunctionName(factory.getFunctionName(this.getClass()));
+    explanation.setImplementingClass(this.getClass().getName());
+    explanation.setExpressionType(ExpressionType.GRAPH_SOURCE);
+    explanation.setExpression(toExpression(factory).toString());
+    
+    // one child is a stream
+    explanation.addChild(tupleStream.toExplanation(factory));
+    
+    // one child is a datastore so add it at this point
+    StreamExplanation child = new StreamExplanation(getStreamNodeId() + "-datastore");
+    child.setFunctionName("solr (graph)");
+    child.setImplementingClass("Solr/Lucene");
+    child.setExpressionType(ExpressionType.DATASTORE);    
+    child.setExpression(queryParams.entrySet().stream().map(e -> String.format(Locale.ROOT, "%s=%s", e.getKey(), e.getValue())).collect(Collectors.joining(",")));    
+    explanation.addChild(child);
+    
+    if(null != metrics){
+      for(Metric metric : metrics){
+          explanation.addHelper(metric.toExplanation(factory));
+      }
+    }
+    
+    return explanation;
+  }
+
 
   public void setStreamContext(StreamContext context) {
     this.traversal = (Traversal) context.get("traversal");
@@ -576,5 +620,15 @@ public class GatherNodesStream extends TupleStream implements Expressible {
         return new Tuple(map);
       }
     }
+    
+    @Override
+    public Explanation toExplanation(StreamFactory factory) throws IOException {
+
+      return new StreamExplanation(getStreamNodeId().toString())
+        .withFunctionName("non-expressible")
+        .withImplementingClass(this.getClass().getName())
+        .withExpressionType(ExpressionType.STREAM_SOURCE)
+        .withExpression("non-expressible");
+    }
   }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d969de41/solr/solrj/src/java/org/apache/solr/client/solrj/io/graph/ShortestPathStream.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/graph/ShortestPathStream.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/graph/ShortestPathStream.java
index bb9b09d..7418e0f 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/graph/ShortestPathStream.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/graph/ShortestPathStream.java
@@ -30,18 +30,22 @@ import java.util.ArrayList;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
+import java.util.stream.Collectors;
 
 import org.apache.solr.client.solrj.io.eq.MultipleFieldEqualitor;
 import org.apache.solr.client.solrj.io.stream.*;
 import org.apache.solr.client.solrj.io.Tuple;
 import org.apache.solr.client.solrj.io.comp.StreamComparator;
 import org.apache.solr.client.solrj.io.eq.FieldEqualitor;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation;
 import org.apache.solr.client.solrj.io.stream.expr.Expressible;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExplanation;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionNamedParameter;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionValue;
 import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
 import org.apache.solr.common.util.ExecutorUtil;
 import org.apache.solr.common.util.SolrjNamedThreadFactory;
 
@@ -61,7 +65,7 @@ public class ShortestPathStream extends TupleStream implements Expressible {
   private boolean found;
   private StreamContext streamContext;
   private int threads;
-  private Map queryParams;
+  private Map<String,String> queryParams;
 
   public ShortestPathStream(String zkHost,
                             String collection,
@@ -221,7 +225,7 @@ public class ShortestPathStream extends TupleStream implements Expressible {
     // collection
     expression.addParameter(collection);
 
-    Set<Map.Entry> entries =  queryParams.entrySet();
+    Set<Map.Entry<String,String>> entries =  queryParams.entrySet();
     // parameters
     for(Map.Entry param : entries){
       String value = param.getValue().toString();
@@ -243,6 +247,27 @@ public class ShortestPathStream extends TupleStream implements Expressible {
     expression.addParameter(new StreamExpressionNamedParameter("edge", fromField+"="+toField));
     return expression;
   }
+  
+  @Override
+  public Explanation toExplanation(StreamFactory factory) throws IOException {
+
+    StreamExplanation explanation = new StreamExplanation(getStreamNodeId().toString());
+    
+    explanation.setFunctionName(factory.getFunctionName(this.getClass()));
+    explanation.setImplementingClass(this.getClass().getName());
+    explanation.setExpressionType(ExpressionType.GRAPH_SOURCE);
+    explanation.setExpression(toExpression(factory).toString());
+    
+    // child is a datastore so add it at this point
+    StreamExplanation child = new StreamExplanation(getStreamNodeId() + "-datastore");
+    child.setFunctionName("solr (graph)");
+    child.setImplementingClass("Solr/Lucene");
+    child.setExpressionType(ExpressionType.DATASTORE);    
+    child.setExpression(queryParams.entrySet().stream().map(e -> String.format(Locale.ROOT, "%s=%s", e.getKey(), e.getValue())).collect(Collectors.joining(",")));    
+    explanation.addChild(child);
+    
+    return explanation;
+  }
 
   public void setStreamContext(StreamContext context) {
     this.streamContext = context;

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d969de41/solr/solrj/src/java/org/apache/solr/client/solrj/io/ops/ConcatOperation.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/ops/ConcatOperation.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/ops/ConcatOperation.java
index a31be43..19d5ef5 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/ops/ConcatOperation.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/ops/ConcatOperation.java
@@ -18,8 +18,11 @@ package org.apache.solr.client.solrj.io.ops;
 
 import java.io.IOException;
 import java.util.Locale;
+import java.util.UUID;
 
 import org.apache.solr.client.solrj.io.Tuple;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionNamedParameter;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
@@ -33,7 +36,8 @@ import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
 public class ConcatOperation implements StreamOperation {
 
   private static final long serialVersionUID = 1;
-
+  private UUID operationNodeId = UUID.randomUUID();
+  
   private String[] fields;
   private String as;
   private String delim;
@@ -94,4 +98,13 @@ public class ConcatOperation implements StreamOperation {
     return expression;
   }
 
+  @Override
+  public Explanation toExplanation(StreamFactory factory) throws IOException {
+    return new Explanation(operationNodeId.toString())
+      .withExpressionType(ExpressionType.OPERATION)
+      .withFunctionName(factory.getFunctionName(getClass()))
+      .withImplementingClass(getClass().getName())
+      .withExpression(toExpression(factory).toString());
+  }
+  
 }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d969de41/solr/solrj/src/java/org/apache/solr/client/solrj/io/ops/DistinctOperation.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/ops/DistinctOperation.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/ops/DistinctOperation.java
index 682fe3b..f4af7b6 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/ops/DistinctOperation.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/ops/DistinctOperation.java
@@ -16,32 +16,20 @@
  */
 package org.apache.solr.client.solrj.io.ops;
 
+import java.io.IOException;
+import java.util.UUID;
+
 import org.apache.solr.client.solrj.io.Tuple;
-import org.apache.solr.client.solrj.io.comp.FieldComparator;
-import org.apache.solr.client.solrj.io.comp.StreamComparator;
-import org.apache.solr.client.solrj.io.eq.FieldEqualitor;
-import org.apache.solr.client.solrj.io.eq.StreamEqualitor;
-import org.apache.solr.client.solrj.io.stream.expr.Expressible;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
-import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionNamedParameter;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
-import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionValue;
 import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
 
-import java.io.IOException;
-import java.io.Serializable;
-import java.util.Comparator;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Locale;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.PriorityQueue;
-
 public class DistinctOperation implements ReduceOperation {
 
   private static final long serialVersionUID = 1L;
+  private UUID operationNodeId = UUID.randomUUID();
   private Tuple current;
 
   public DistinctOperation(StreamExpression expression, StreamFactory factory) throws IOException {
@@ -60,6 +48,15 @@ public class DistinctOperation implements ReduceOperation {
     return expression;
   }
 
+  @Override
+  public Explanation toExplanation(StreamFactory factory) throws IOException {
+    return new Explanation(operationNodeId.toString())
+      .withExpressionType(ExpressionType.OPERATION)
+      .withFunctionName(factory.getFunctionName(getClass()))
+      .withImplementingClass(getClass().getName())
+      .withExpression(toExpression(factory).toString());
+  }
+
   public Tuple reduce() {
     // Return the tuple after setting current to null. This will ensure the next call to 
     // operate stores that tuple

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d969de41/solr/solrj/src/java/org/apache/solr/client/solrj/io/ops/GroupOperation.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/ops/GroupOperation.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/ops/GroupOperation.java
index 1687352..9ed5cbe 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/ops/GroupOperation.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/ops/GroupOperation.java
@@ -16,29 +16,34 @@
  */
 package org.apache.solr.client.solrj.io.ops;
 
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.PriorityQueue;
+import java.util.UUID;
+
 import org.apache.solr.client.solrj.io.Tuple;
 import org.apache.solr.client.solrj.io.comp.FieldComparator;
 import org.apache.solr.client.solrj.io.comp.StreamComparator;
-import org.apache.solr.client.solrj.io.stream.expr.Expressible;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionNamedParameter;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionValue;
 import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
 
-import java.io.IOException;
-import java.io.Serializable;
-import java.util.Comparator;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Locale;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.PriorityQueue;
-
 public class GroupOperation implements ReduceOperation {
 
+  private static final long serialVersionUID = 1L;
+  private UUID operationNodeId = UUID.randomUUID();
+  
   private PriorityQueue<Tuple> priorityQueue;
   private Comparator comp;
   private StreamComparator streamComparator;
@@ -86,6 +91,18 @@ public class GroupOperation implements ReduceOperation {
     return expression;
   }
 
+  @Override
+  public Explanation toExplanation(StreamFactory factory) throws IOException {
+    return new Explanation(operationNodeId.toString())
+      .withExpressionType(ExpressionType.OPERATION)
+      .withFunctionName(factory.getFunctionName(getClass()))
+      .withImplementingClass(getClass().getName())
+      .withExpression(toExpression(factory).toString())
+      .withHelpers(new Explanation[]{
+          streamComparator.toExplanation(factory)
+      });
+  }
+
   public Tuple reduce() {
     Map map = new HashMap();
     List<Map> list = new ArrayList();
@@ -122,6 +139,7 @@ public class GroupOperation implements ReduceOperation {
     }
 
     public int compare(Tuple t1, Tuple t2) {
+      // Couldn't this be comp.compare(t2,t1) ?
       return comp.compare(t1, t2)*(-1);
     }
   }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d969de41/solr/solrj/src/java/org/apache/solr/client/solrj/io/ops/ReplaceOperation.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/ops/ReplaceOperation.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/ops/ReplaceOperation.java
index e569c0d..27342dd 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/ops/ReplaceOperation.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/ops/ReplaceOperation.java
@@ -20,6 +20,7 @@ import java.io.IOException;
 import java.util.Locale;
 
 import org.apache.solr.client.solrj.io.Tuple;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionNamedParameter;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
@@ -80,4 +81,9 @@ public class ReplaceOperation implements StreamOperation {
     return replacer.toExpression(factory);
   }
   
+  @Override 
+  public Explanation toExplanation(StreamFactory factory) throws IOException{
+    return replacer.toExplanation(factory);
+  }
+  
 }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d969de41/solr/solrj/src/java/org/apache/solr/client/solrj/io/ops/ReplaceWithFieldOperation.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/ops/ReplaceWithFieldOperation.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/ops/ReplaceWithFieldOperation.java
index 0dcdece..99b808e 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/ops/ReplaceWithFieldOperation.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/ops/ReplaceWithFieldOperation.java
@@ -18,13 +18,16 @@ package org.apache.solr.client.solrj.io.ops;
 
 import java.io.IOException;
 import java.util.Locale;
+import java.util.UUID;
 
 import org.apache.solr.client.solrj.io.Tuple;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionNamedParameter;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionValue;
 import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
 
 /**
  * Implementation of replace(...., withField=fieldName)
@@ -33,6 +36,7 @@ import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
 public class ReplaceWithFieldOperation implements StreamOperation {
 
   private static final long serialVersionUID = 1;
+  private UUID operationNodeId = UUID.randomUUID();
   
   private boolean wasBuiltWithFieldName;
   private String originalFieldName;
@@ -108,4 +112,12 @@ public class ReplaceWithFieldOperation implements StreamOperation {
     return expression;
   }
   
+  @Override
+  public Explanation toExplanation(StreamFactory factory) throws IOException {
+    return new Explanation(operationNodeId.toString())
+      .withExpressionType(ExpressionType.OPERATION)
+      .withFunctionName(factory.getFunctionName(getClass()))
+      .withImplementingClass(getClass().getName())
+      .withExpression(toExpression(factory).toString());
+  }
 }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d969de41/solr/solrj/src/java/org/apache/solr/client/solrj/io/ops/ReplaceWithValueOperation.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/ops/ReplaceWithValueOperation.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/ops/ReplaceWithValueOperation.java
index 8b74e2a..0301def 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/ops/ReplaceWithValueOperation.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/ops/ReplaceWithValueOperation.java
@@ -18,13 +18,16 @@ package org.apache.solr.client.solrj.io.ops;
 
 import java.io.IOException;
 import java.util.Locale;
+import java.util.UUID;
 
 import org.apache.solr.client.solrj.io.Tuple;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionNamedParameter;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionValue;
 import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
 
 /**
  * Implementation of replace(...., withValue="some value")
@@ -33,6 +36,7 @@ import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
 public class ReplaceWithValueOperation implements StreamOperation {
 
   private static final long serialVersionUID = 1;
+  private UUID operationNodeId = UUID.randomUUID();
   
   private boolean wasBuiltWithFieldName;
   private String fieldName;
@@ -113,4 +117,12 @@ public class ReplaceWithValueOperation implements StreamOperation {
     return expression;
   }
   
+  @Override
+  public Explanation toExplanation(StreamFactory factory) throws IOException {
+    return new Explanation(operationNodeId.toString())
+      .withExpressionType(ExpressionType.OPERATION)
+      .withFunctionName(factory.getFunctionName(getClass()))
+      .withImplementingClass(getClass().getName())
+      .withExpression(toExpression(factory).toString());
+  }
 }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d969de41/solr/solrj/src/java/org/apache/solr/client/solrj/io/ops/StreamOperation.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/ops/StreamOperation.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/ops/StreamOperation.java
index 643628f..1f71901 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/ops/StreamOperation.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/ops/StreamOperation.java
@@ -16,13 +16,10 @@
  */
 package org.apache.solr.client.solrj.io.ops;
 
-import java.io.IOException;
 import java.io.Serializable;
 
 import org.apache.solr.client.solrj.io.Tuple;
 import org.apache.solr.client.solrj.io.stream.expr.Expressible;
-import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
-import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
 
 /**
  * Interface for any operation one can perform on a tuple in a TupleStream

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d969de41/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/CloudSolrStream.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/CloudSolrStream.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/CloudSolrStream.java
index c9c7b22..b619f2f 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/CloudSolrStream.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/CloudSolrStream.java
@@ -26,12 +26,13 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Map.Entry;
-import java.util.Set;
 import java.util.Random;
+import java.util.Set;
 import java.util.TreeSet;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
+import java.util.stream.Collectors;
 
 import org.apache.solr.client.solrj.impl.CloudSolrClient;
 import org.apache.solr.client.solrj.impl.CloudSolrClient.Builder;
@@ -41,10 +42,12 @@ import org.apache.solr.client.solrj.io.comp.ComparatorOrder;
 import org.apache.solr.client.solrj.io.comp.FieldComparator;
 import org.apache.solr.client.solrj.io.comp.MultipleFieldComparator;
 import org.apache.solr.client.solrj.io.comp.StreamComparator;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
 import org.apache.solr.client.solrj.io.stream.expr.Expressible;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExplanation;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionNamedParameter;
-import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionValue;
 import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
 import org.apache.solr.common.cloud.ClusterState;
@@ -154,7 +157,7 @@ public class CloudSolrStream extends TupleStream implements Expressible {
   }
   
   @Override
-  public StreamExpressionParameter toExpression(StreamFactory factory) throws IOException {
+  public StreamExpression toExpression(StreamFactory factory) throws IOException {
     // functionName(collectionName, param1, param2, ..., paramN, sort="comp", [aliases="field=alias,..."])
     
     // function name
@@ -194,6 +197,29 @@ public class CloudSolrStream extends TupleStream implements Expressible {
     return expression;   
   }
   
+  @Override
+  public Explanation toExplanation(StreamFactory factory) throws IOException {
+
+    StreamExplanation explanation = new StreamExplanation(getStreamNodeId().toString());
+    
+    explanation.setFunctionName(factory.getFunctionName(this.getClass()));
+    explanation.setImplementingClass(this.getClass().getName());
+    explanation.setExpressionType(ExpressionType.STREAM_SOURCE);
+    explanation.setExpression(toExpression(factory).toString());
+    
+    // child is a datastore so add it at this point
+    StreamExplanation child = new StreamExplanation(getStreamNodeId() + "-datastore");
+    child.setFunctionName(String.format(Locale.ROOT, "solr (%s)", collection));
+    child.setImplementingClass("Solr/Lucene");
+    child.setExpressionType(ExpressionType.DATASTORE);
+    if(null != params){
+      child.setExpression(params.entrySet().stream().map(e -> String.format(Locale.ROOT, "%s=%s", e.getKey(), e.getValue())).collect(Collectors.joining(",")));
+    }
+    explanation.addChild(child);
+    
+    return explanation;
+  }
+  
   private void init(String collectionName, String zkHost, Map params) throws IOException {
     this.zkHost = zkHost;
     this.collection = collectionName;

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d969de41/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/ComplementStream.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/ComplementStream.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/ComplementStream.java
index bd3c233..a47f05c 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/ComplementStream.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/ComplementStream.java
@@ -22,12 +22,13 @@ import java.util.List;
 import java.util.Locale;
 
 import org.apache.solr.client.solrj.io.Tuple;
-import org.apache.solr.client.solrj.io.comp.FieldComparator;
 import org.apache.solr.client.solrj.io.comp.StreamComparator;
 import org.apache.solr.client.solrj.io.eq.FieldEqualitor;
 import org.apache.solr.client.solrj.io.eq.StreamEqualitor;
-import org.apache.solr.client.solrj.io.ops.DistinctOperation;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
 import org.apache.solr.client.solrj.io.stream.expr.Expressible;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExplanation;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionNamedParameter;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionValue;
@@ -88,23 +89,33 @@ public class ComplementStream extends TupleStream implements Expressible {
   }
   
   @Override
-  public StreamExpression toExpression(StreamFactory factory) throws IOException {    
+  public StreamExpression toExpression(StreamFactory factory) throws IOException{
+    return toExpression(factory, true);
+  }
+  
+  private StreamExpression toExpression(StreamFactory factory, boolean includeStreams) throws IOException {
     // function name
     StreamExpression expression = new StreamExpression(factory.getFunctionName(this.getClass()));
     
-    // streams
-    if(streamA instanceof Expressible){
-      expression.addParameter(((Expressible)streamA).toExpression(factory));
-    }
-    else{
-      throw new IOException("This IntersectionStream contains a non-expressible TupleStream - it cannot be converted to an expression");
-    }
-    
-    if(originalStreamB instanceof Expressible){
-      expression.addParameter(((Expressible)originalStreamB).toExpression(factory));
+    if(includeStreams){
+      // streams
+      if(streamA instanceof Expressible){
+        expression.addParameter(((Expressible)streamA).toExpression(factory));
+      }
+      else{
+        throw new IOException("This IntersectionStream contains a non-expressible TupleStream - it cannot be converted to an expression");
+      }
+      
+      if(originalStreamB instanceof Expressible){
+        expression.addParameter(((Expressible)originalStreamB).toExpression(factory));
+      }
+      else{
+        throw new IOException("This IntersectStream contains a non-expressible TupleStream - it cannot be converted to an expression");
+      }
     }
     else{
-      throw new IOException("This IntersectStream contains a non-expressible TupleStream - it cannot be converted to an expression");
+      expression.addParameter("<stream>");
+      expression.addParameter("<stream>");
     }
     
     // on
@@ -112,6 +123,22 @@ public class ComplementStream extends TupleStream implements Expressible {
     
     return expression;   
   }
+  
+  @Override
+  public Explanation toExplanation(StreamFactory factory) throws IOException {
+
+    return new StreamExplanation(getStreamNodeId().toString())
+      .withChildren(new Explanation[]{
+        streamA.toExplanation(factory),
+        originalStreamB.toExplanation(factory)
+      })
+      .withFunctionName(factory.getFunctionName(this.getClass()))
+      .withImplementingClass(this.getClass().getName())
+      .withExpressionType(ExpressionType.STREAM_DECORATOR)
+      .withExpression(toExpression(factory, false).toString())
+      .withHelper(eq.toExplanation(factory))
+      ;    
+  }
 
   public void setStreamContext(StreamContext context) {
     this.streamA.setStreamContext(context);
@@ -119,7 +146,7 @@ public class ComplementStream extends TupleStream implements Expressible {
   }
 
   public List<TupleStream> children() {
-    List<TupleStream> l =  new ArrayList();
+    List<TupleStream> l =  new ArrayList<TupleStream>();
     l.add(streamA);
     l.add(streamB);
     return l;

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d969de41/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/DaemonStream.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/DaemonStream.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/DaemonStream.java
index 752ea7c..77648df 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/DaemonStream.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/DaemonStream.java
@@ -15,23 +15,26 @@
  * limitations under the License.
  */
 package org.apache.solr.client.solrj.io.stream;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.concurrent.ArrayBlockingQueue;
+
 import org.apache.solr.client.solrj.io.Tuple;
 import org.apache.solr.client.solrj.io.comp.StreamComparator;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
 import org.apache.solr.client.solrj.io.stream.expr.Expressible;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExplanation;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionNamedParameter;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionValue;
 import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.lang.invoke.MethodHandles;
-import java.util.Locale;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.Date;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.List;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -100,15 +103,24 @@ public class DaemonStream extends TupleStream implements Expressible {
   }
 
   @Override
-  public StreamExpression toExpression(StreamFactory factory) throws IOException {
+  public StreamExpression toExpression(StreamFactory factory) throws IOException{
+    return toExpression(factory, true);
+  }
+  
+  private StreamExpression toExpression(StreamFactory factory, boolean includeStreams) throws IOException {
     // function name
     StreamExpression expression = new StreamExpression(factory.getFunctionName(this.getClass()));
 
-    // streams
-    if(tupleStream instanceof Expressible){
-      expression.addParameter(((Expressible)tupleStream).toExpression(factory));
-    } else {
-      throw new IOException("This UniqueStream contains a non-expressible TupleStream - it cannot be converted to an expression");
+    if(includeStreams){
+      // streams
+      if(tupleStream instanceof Expressible){
+        expression.addParameter(((Expressible)tupleStream).toExpression(factory));
+      } else {
+        throw new IOException("This UniqueStream contains a non-expressible TupleStream - it cannot be converted to an expression");
+      }
+    }
+    else{
+      expression.addParameter("<stream>");
     }
 
     expression.addParameter(new StreamExpressionNamedParameter("id", id));
@@ -117,6 +129,19 @@ public class DaemonStream extends TupleStream implements Expressible {
 
     return expression;
   }
+  
+  @Override
+  public Explanation toExplanation(StreamFactory factory) throws IOException {
+
+    return new StreamExplanation(getStreamNodeId().toString())
+      .withChildren(new Explanation[] {
+        tupleStream.toExplanation(factory)
+      })
+      .withFunctionName(factory.getFunctionName(this.getClass()))
+      .withImplementingClass(this.getClass().getName())
+      .withExpressionType(ExpressionType.STREAM_DECORATOR)
+      .withExpression(toExpression(factory, false).toString());
+  }
 
   public int remainingCapacity() {
     return this.queue.remainingCapacity();

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d969de41/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/ExceptionStream.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/ExceptionStream.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/ExceptionStream.java
index e442d1b..4604105 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/ExceptionStream.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/ExceptionStream.java
@@ -24,6 +24,10 @@ import java.util.Map;
 
 import org.apache.solr.client.solrj.io.Tuple;
 import org.apache.solr.client.solrj.io.comp.StreamComparator;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExplanation;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
 import org.apache.solr.common.SolrException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -70,6 +74,16 @@ public class ExceptionStream extends TupleStream {
       return new Tuple(fields);
     }
   }
+  
+  @Override
+  public Explanation toExplanation(StreamFactory factory) throws IOException {
+
+    return new StreamExplanation(getStreamNodeId().toString())
+      .withFunctionName("non-expressible")
+      .withImplementingClass(this.getClass().getName())
+      .withExpressionType(ExpressionType.STREAM_SOURCE)
+      .withExpression("non-expressible");
+  }
 
   public StreamComparator getStreamSort() {
     return this.stream.getStreamSort();

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d969de41/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/FacetStream.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/FacetStream.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/FacetStream.java
index 3474257..ceaf13c 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/FacetStream.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/FacetStream.java
@@ -17,13 +17,14 @@
 package org.apache.solr.client.solrj.io.stream;
 
 import java.io.IOException;
-import java.util.HashMap;
 import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
-import java.util.Collections;
 import java.util.Map.Entry;
+import java.util.stream.Collectors;
 
 import org.apache.solr.client.solrj.impl.CloudSolrClient;
 import org.apache.solr.client.solrj.impl.CloudSolrClient.Builder;
@@ -33,7 +34,10 @@ import org.apache.solr.client.solrj.io.comp.ComparatorOrder;
 import org.apache.solr.client.solrj.io.comp.FieldComparator;
 import org.apache.solr.client.solrj.io.comp.MultipleFieldComparator;
 import org.apache.solr.client.solrj.io.comp.StreamComparator;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
 import org.apache.solr.client.solrj.io.stream.expr.Expressible;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExplanation;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionNamedParameter;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
@@ -58,7 +62,7 @@ public class FacetStream extends TupleStream implements Expressible  {
   private Metric[] metrics;
   private int bucketSizeLimit;
   private FieldComparator[] bucketSorts;
-  private List<Tuple> tuples = new ArrayList();
+  private List<Tuple> tuples = new ArrayList<Tuple>();
   private int index;
   private String zkHost;
   private Map<String, String> props;
@@ -267,7 +271,31 @@ public class FacetStream extends TupleStream implements Expressible  {
     return expression;   
   }
 
+  @Override
+  public Explanation toExplanation(StreamFactory factory) throws IOException {
 
+    StreamExplanation explanation = new StreamExplanation(getStreamNodeId().toString());
+    
+    explanation.setFunctionName(factory.getFunctionName(this.getClass()));
+    explanation.setImplementingClass(this.getClass().getName());
+    explanation.setExpressionType(ExpressionType.STREAM_SOURCE);
+    explanation.setExpression(toExpression(factory).toString());
+    
+    // child is a datastore so add it at this point
+    StreamExplanation child = new StreamExplanation(getStreamNodeId() + "-datastore");
+    child.setFunctionName(String.format(Locale.ROOT, "solr (%s)", collection)); 
+    // TODO: fix this so we know the # of workers - check with Joel about a Topic's ability to be in a
+    // parallel stream.
+    
+    child.setImplementingClass("Solr/Lucene");
+    child.setExpressionType(ExpressionType.DATASTORE);    
+    child.setExpression(props.entrySet().stream().map(e -> String.format(Locale.ROOT, "%s=%s", e.getKey(), e.getValue())).collect(Collectors.joining(",")));
+    
+    explanation.addChild(child);
+    
+    return explanation;
+  }
+  
   public void setStreamContext(StreamContext context) {
     cache = context.getSolrClientCache();
   }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d969de41/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/HashJoinStream.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/HashJoinStream.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/HashJoinStream.java
index 717e8e5..dfb678f 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/HashJoinStream.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/HashJoinStream.java
@@ -24,7 +24,10 @@ import java.util.Locale;
 
 import org.apache.solr.client.solrj.io.Tuple;
 import org.apache.solr.client.solrj.io.comp.StreamComparator;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
 import org.apache.solr.client.solrj.io.stream.expr.Expressible;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExplanation;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionNamedParameter;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionValue;
@@ -99,17 +102,27 @@ public class HashJoinStream extends TupleStream implements Expressible {
   }
   
   @Override
-  public StreamExpression toExpression(StreamFactory factory) throws IOException {    
+  public StreamExpression toExpression(StreamFactory factory) throws IOException{
+    return toExpression(factory, true);
+  }
+  
+  private StreamExpression toExpression(StreamFactory factory, boolean includeStreams) throws IOException {
     // function name
     StreamExpression expression = new StreamExpression(factory.getFunctionName(this.getClass()));
     
-    // streams
-    if(hashStream instanceof Expressible && fullStream instanceof Expressible){
-      expression.addParameter(((Expressible)fullStream).toExpression(factory));
-      expression.addParameter(new StreamExpressionNamedParameter("hashed", ((Expressible)hashStream).toExpression(factory)));
+    if(includeStreams){
+      // streams
+      if(hashStream instanceof Expressible && fullStream instanceof Expressible){
+        expression.addParameter(((Expressible)fullStream).toExpression(factory));
+        expression.addParameter(new StreamExpressionNamedParameter("hashed", ((Expressible)hashStream).toExpression(factory)));
+      }
+      else{
+        throw new IOException("This HashJoinStream contains a non-expressible TupleStream - it cannot be converted to an expression");
+      }
     }
     else{
-      throw new IOException("This HashJoinStream contains a non-expressible TupleStream - it cannot be converted to an expression");
+      expression.addParameter("<stream>");
+      expression.addParameter("hashed=<stream>");
     }
     
     // on
@@ -122,6 +135,20 @@ public class HashJoinStream extends TupleStream implements Expressible {
     
     return expression;   
   }
+  
+  @Override
+  public Explanation toExplanation(StreamFactory factory) throws IOException {
+
+    return new StreamExplanation(getStreamNodeId().toString())
+      .withChildren(new Explanation[]{
+        fullStream.toExplanation(factory),
+        hashStream.toExplanation(factory)
+      })
+      .withFunctionName(factory.getFunctionName(this.getClass()))
+      .withImplementingClass(this.getClass().getName())
+      .withExpressionType(ExpressionType.STREAM_DECORATOR)
+      .withExpression(toExpression(factory, false).toString());    
+  }
 
   public void setStreamContext(StreamContext context) {
     this.hashStream.setStreamContext(context);
@@ -129,7 +156,7 @@ public class HashJoinStream extends TupleStream implements Expressible {
   }
 
   public List<TupleStream> children() {
-    List<TupleStream> l =  new ArrayList();
+    List<TupleStream> l =  new ArrayList<TupleStream>();
     l.add(hashStream);
     l.add(fullStream);
     return l;

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d969de41/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/IntersectStream.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/IntersectStream.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/IntersectStream.java
index 273ca60..3c04e46 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/IntersectStream.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/IntersectStream.java
@@ -22,12 +22,13 @@ import java.util.List;
 import java.util.Locale;
 
 import org.apache.solr.client.solrj.io.Tuple;
-import org.apache.solr.client.solrj.io.comp.FieldComparator;
 import org.apache.solr.client.solrj.io.comp.StreamComparator;
 import org.apache.solr.client.solrj.io.eq.FieldEqualitor;
 import org.apache.solr.client.solrj.io.eq.StreamEqualitor;
-import org.apache.solr.client.solrj.io.ops.DistinctOperation;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
 import org.apache.solr.client.solrj.io.stream.expr.Expressible;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExplanation;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionNamedParameter;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionValue;
@@ -88,23 +89,33 @@ public class IntersectStream extends TupleStream implements Expressible {
   }
   
   @Override
-  public StreamExpression toExpression(StreamFactory factory) throws IOException {    
+  public StreamExpression toExpression(StreamFactory factory) throws IOException{
+    return toExpression(factory, true);
+  }
+  
+  private StreamExpression toExpression(StreamFactory factory, boolean includeStreams) throws IOException {
     // function name
     StreamExpression expression = new StreamExpression(factory.getFunctionName(this.getClass()));
     
-    // streams
-    if(streamA instanceof Expressible){
-      expression.addParameter(((Expressible)streamA).toExpression(factory));
-    }
-    else{
-      throw new IOException("This IntersectStream contains a non-expressible TupleStream - it cannot be converted to an expression");
-    }
-    
-    if(originalStreamB instanceof Expressible){
-      expression.addParameter(((Expressible)originalStreamB).toExpression(factory));
+    if(includeStreams){
+      // streams
+      if(streamA instanceof Expressible){
+        expression.addParameter(((Expressible)streamA).toExpression(factory));
+      }
+      else{
+        throw new IOException("This IntersectStream contains a non-expressible TupleStream - it cannot be converted to an expression");
+      }
+      
+      if(originalStreamB instanceof Expressible){
+        expression.addParameter(((Expressible)originalStreamB).toExpression(factory));
+      }
+      else{
+        throw new IOException("This IntersectStream contains a non-expressible TupleStream - it cannot be converted to an expression");
+      }
     }
     else{
-      throw new IOException("This IntersectStream contains a non-expressible TupleStream - it cannot be converted to an expression");
+      expression.addParameter("<stream>");
+      expression.addParameter("<stream>");
     }
     
     // on
@@ -113,13 +124,28 @@ public class IntersectStream extends TupleStream implements Expressible {
     return expression;   
   }
 
+  @Override
+  public Explanation toExplanation(StreamFactory factory) throws IOException {
+
+    return new StreamExplanation(getStreamNodeId().toString())
+      .withChildren(new Explanation[]{
+        streamA.toExplanation(factory),
+        originalStreamB.toExplanation(factory)
+      })
+      .withFunctionName(factory.getFunctionName(this.getClass()))
+      .withImplementingClass(this.getClass().getName())
+      .withExpressionType(ExpressionType.STREAM_DECORATOR)
+      .withExpression(toExpression(factory, false).toString())
+      .withHelper(eq.toExplanation(factory));    
+  }
+  
   public void setStreamContext(StreamContext context) {
     this.streamA.setStreamContext(context);
     this.streamB.setStreamContext(context);
   }
 
   public List<TupleStream> children() {
-    List<TupleStream> l =  new ArrayList();
+    List<TupleStream> l =  new ArrayList<TupleStream>();
     l.add(streamA);
     l.add(streamB);
     return l;

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d969de41/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/JDBCStream.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/JDBCStream.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/JDBCStream.java
index a98f2e4..d1a301e 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/JDBCStream.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/JDBCStream.java
@@ -34,7 +34,10 @@ import java.util.Properties;
 import org.apache.solr.client.solrj.io.Tuple;
 import org.apache.solr.client.solrj.io.comp.FieldComparator;
 import org.apache.solr.client.solrj.io.comp.StreamComparator;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
 import org.apache.solr.client.solrj.io.stream.expr.Expressible;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExplanation;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionNamedParameter;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
@@ -349,6 +352,40 @@ public class JDBCStream extends TupleStream implements Expressible {
   }
   
   @Override
+  public Explanation toExplanation(StreamFactory factory) throws IOException {
+
+    StreamExplanation explanation = new StreamExplanation(getStreamNodeId().toString());
+    
+    explanation.setFunctionName(factory.getFunctionName(this.getClass()));
+    explanation.setImplementingClass(this.getClass().getName());
+    explanation.setExpressionType(ExpressionType.STREAM_SOURCE);
+    
+    StreamExpression expression = (StreamExpression)toExpression(factory);
+    explanation.setExpression(expression.toString());
+    
+    String driverClassName = this.driverClassName;
+    if(null == driverClassName){
+      try{
+        driverClassName = DriverManager.getDriver(connectionUrl).getClass().getName();
+      }
+      catch(Exception e){
+        driverClassName = String.format(Locale.ROOT, "Failed to find driver for connectionUrl='%s'", connectionUrl);
+      }
+    }
+    
+    // child is a datastore so add it at this point
+    StreamExplanation child = new StreamExplanation(getStreamNodeId() + "-datastore");
+    child.setFunctionName("jdbc-source");
+    child.setImplementingClass(driverClassName);
+    child.setExpressionType(ExpressionType.DATASTORE);    
+    child.setExpression(sqlQuery);
+    
+    explanation.addChild(child);
+    
+    return explanation;
+  }
+  
+  @Override
   public List<TupleStream> children() {
     return new ArrayList<TupleStream>();
   }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d969de41/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/JoinStream.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/JoinStream.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/JoinStream.java
index 69df463..acb7fe9 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/JoinStream.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/JoinStream.java
@@ -26,7 +26,10 @@ import org.apache.solr.client.solrj.io.Tuple;
 import org.apache.solr.client.solrj.io.comp.StreamComparator;
 import org.apache.solr.client.solrj.io.eq.FieldEqualitor;
 import org.apache.solr.client.solrj.io.eq.StreamEqualitor;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
 import org.apache.solr.client.solrj.io.stream.expr.Expressible;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExplanation;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionNamedParameter;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionValue;
@@ -87,13 +90,22 @@ public abstract class JoinStream extends TupleStream implements Expressible {
   }
   
   @Override
-  public StreamExpression toExpression(StreamFactory factory) throws IOException {
+  public StreamExpression toExpression(StreamFactory factory) throws IOException{
+    return toExpression(factory, true);
+  }
+  
+  private StreamExpression toExpression(StreamFactory factory, boolean includeStreams) throws IOException {
     // function name
     StreamExpression expression = new StreamExpression(factory.getFunctionName(this.getClass()));
     
     // streams
     for (PushBackStream stream : streams) {
-      expression.addParameter(stream.toExpression(factory));
+      if(includeStreams){
+        expression.addParameter(stream.toExpression(factory));
+      }
+      else{
+        expression.addParameter("<stream>");
+      }
     }
     
     // on
@@ -107,6 +119,23 @@ public abstract class JoinStream extends TupleStream implements Expressible {
     return expression;
   }
   
+  @Override
+  public Explanation toExplanation(StreamFactory factory) throws IOException {
+
+    StreamExplanation explanation = new StreamExplanation(getStreamNodeId().toString());
+    explanation.setFunctionName(factory.getFunctionName(this.getClass()));
+    explanation.setImplementingClass(this.getClass().getName());
+    explanation.setExpressionType(ExpressionType.STREAM_DECORATOR);
+    explanation.setExpression(toExpression(factory, false).toString());
+    explanation.addHelper(eq.toExplanation(factory));
+    
+    for(TupleStream stream : streams){
+      explanation.addChild(stream.toExplanation(factory));
+    }
+    
+    return explanation;    
+  }
+  
   public void setStreamContext(StreamContext context) {
     for (PushBackStream stream : streams) {
       stream.setStreamContext(context);

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d969de41/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/MergeStream.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/MergeStream.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/MergeStream.java
index 2ea926c..c50ac77 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/MergeStream.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/MergeStream.java
@@ -24,7 +24,10 @@ import java.util.Locale;
 import org.apache.solr.client.solrj.io.Tuple;
 import org.apache.solr.client.solrj.io.comp.FieldComparator;
 import org.apache.solr.client.solrj.io.comp.StreamComparator;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
 import org.apache.solr.client.solrj.io.stream.expr.Expressible;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExplanation;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionNamedParameter;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionValue;
@@ -95,13 +98,22 @@ public class MergeStream extends TupleStream implements Expressible {
   }
   
   @Override
-  public StreamExpression toExpression(StreamFactory factory) throws IOException {    
+  public StreamExpression toExpression(StreamFactory factory) throws IOException{
+    return toExpression(factory, true);
+  }
+  
+  private StreamExpression toExpression(StreamFactory factory, boolean includeStreams) throws IOException {
     // function name
     StreamExpression expression = new StreamExpression(factory.getFunctionName(this.getClass()));
     
     // streams
     for(PushBackStream stream : streams){
-      expression.addParameter(stream.toExpression(factory));
+      if(includeStreams){
+        expression.addParameter(stream.toExpression(factory));
+      }
+      else{
+        expression.addParameter("<stream>");
+      }
     }
     
     // on
@@ -109,6 +121,23 @@ public class MergeStream extends TupleStream implements Expressible {
     
     return expression;   
   }
+  
+  @Override
+  public Explanation toExplanation(StreamFactory factory) throws IOException {
+
+    StreamExplanation explanation = new StreamExplanation(getStreamNodeId().toString());
+    explanation.setFunctionName(factory.getFunctionName(this.getClass()));
+    explanation.setImplementingClass(this.getClass().getName());
+    explanation.setExpressionType(ExpressionType.STREAM_DECORATOR);
+    explanation.setExpression(toExpression(factory, false).toString());
+    explanation.addHelper(comp.toExplanation(factory));
+    
+    for(PushBackStream stream : streams){
+      explanation.addChild(stream.toExplanation(factory));
+    }
+    
+    return explanation;    
+  }
 
   public void setStreamContext(StreamContext context) {
     for(PushBackStream stream : streams){
@@ -117,7 +146,7 @@ public class MergeStream extends TupleStream implements Expressible {
   }
 
   public List<TupleStream> children() {
-    List<TupleStream> l =  new ArrayList();
+    List<TupleStream> l =  new ArrayList<TupleStream>();
     for(PushBackStream stream : streams){
       l.add(stream);
     }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d969de41/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/ParallelStream.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/ParallelStream.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/ParallelStream.java
index 28b1c6e..9570643 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/ParallelStream.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/ParallelStream.java
@@ -30,16 +30,20 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.stream.Collectors;
 import java.util.Random;
 
 import org.apache.solr.client.solrj.io.Tuple;
 import org.apache.solr.client.solrj.io.comp.FieldComparator;
 import org.apache.solr.client.solrj.io.comp.StreamComparator;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation;
 import org.apache.solr.client.solrj.io.stream.expr.Expressible;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExplanation;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionNamedParameter;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionValue;
 import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
 import org.apache.solr.common.cloud.ClusterState;
 import org.apache.solr.common.cloud.Replica;
 import org.apache.solr.common.cloud.Slice;
@@ -163,8 +167,11 @@ public class ParallelStream extends CloudSolrStream implements Expressible {
   }
 
   @Override
-  public StreamExpression toExpression(StreamFactory factory) throws IOException {    
-
+  public StreamExpression toExpression(StreamFactory factory) throws IOException{
+    return toExpression(factory, true);
+  }
+  
+  private StreamExpression toExpression(StreamFactory factory, boolean includeStreams) throws IOException {
     // function name
     StreamExpression expression = new StreamExpression(factory.getFunctionName(this.getClass()));
     
@@ -174,12 +181,16 @@ public class ParallelStream extends CloudSolrStream implements Expressible {
     // workers
     expression.addParameter(new StreamExpressionNamedParameter("workers", Integer.toString(workers)));
     
-    // stream
-    if(tupleStream instanceof Expressible){
-      expression.addParameter(((Expressible)tupleStream).toExpression(factory));
+    if(includeStreams){
+      if(tupleStream instanceof Expressible){
+        expression.addParameter(((Expressible)tupleStream).toExpression(factory));
+      }
+      else{
+        throw new IOException("This ParallelStream contains a non-expressible TupleStream - it cannot be converted to an expression");
+      }
     }
     else{
-      throw new IOException("This ParallelStream contains a non-expressible TupleStream - it cannot be converted to an expression");
+      expression.addParameter("<stream>");
     }
         
     // sort
@@ -191,6 +202,24 @@ public class ParallelStream extends CloudSolrStream implements Expressible {
     return expression;   
   }
   
+  @Override
+  public Explanation toExplanation(StreamFactory factory) throws IOException {
+
+    StreamExplanation explanation = new StreamExplanation(getStreamNodeId().toString());
+    
+    explanation.setFunctionName(factory.getFunctionName(this.getClass()));
+    explanation.setImplementingClass(this.getClass().getName());
+    explanation.setExpressionType(ExpressionType.STREAM_DECORATOR);
+    explanation.setExpression(toExpression(factory, false).toString());
+    
+    // add a child for each worker
+    for(int idx = 0; idx < workers; ++idx){
+      explanation.addChild(tupleStream.toExplanation(factory));
+    }
+    
+    return explanation;
+  }
+  
   public List<TupleStream> children() {
     List l = new ArrayList();
     l.add(tupleStream);


Mime
View raw message