lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From va...@apache.org
Subject [3/3] lucene-solr:master: SOLR-11598: Support more than 4 sort fields in the export writer
Date Fri, 20 Jul 2018 19:23:09 GMT
SOLR-11598: Support more than 4 sort fields in the export writer


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/9d9c3a0c
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/9d9c3a0c
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/9d9c3a0c

Branch: refs/heads/master
Commit: 9d9c3a0cd87832980a4745ec96fb2cd1216dcb4e
Parents: 509561b
Author: Varun Thacker <varun@apache.org>
Authored: Fri Jul 20 11:38:02 2018 -0700
Committer: Varun Thacker <varun@apache.org>
Committed: Fri Jul 20 11:38:07 2018 -0700

----------------------------------------------------------------------
 solr/CHANGES.txt                                |    4 +
 .../org/apache/solr/handler/ExportHandler.java  |    1 +
 .../org/apache/solr/handler/ExportWriter.java   | 1764 ------------------
 .../solr/handler/export/BoolFieldWriter.java    |   52 +
 .../solr/handler/export/DateFieldWriter.java    |   46 +
 .../apache/solr/handler/export/DoubeValue.java  |   81 +
 .../apache/solr/handler/export/DoubleCmp.java   |   43 +
 .../solr/handler/export/DoubleFieldWriter.java  |   45 +
 .../solr/handler/export/ExportWriter.java       |  448 +++++
 .../apache/solr/handler/export/FieldWriter.java |   27 +
 .../apache/solr/handler/export/FloatCmp.java    |   44 +
 .../solr/handler/export/FloatFieldWriter.java   |   45 +
 .../apache/solr/handler/export/FloatValue.java  |   78 +
 .../org/apache/solr/handler/export/IntComp.java |   45 +
 .../solr/handler/export/IntFieldWriter.java     |   45 +
 .../apache/solr/handler/export/IntValue.java    |   77 +
 .../org/apache/solr/handler/export/LongCmp.java |   45 +
 .../solr/handler/export/LongFieldWriter.java    |   45 +
 .../apache/solr/handler/export/LongValue.java   |   78 +
 .../solr/handler/export/MultiFieldWriter.java   |  104 ++
 .../solr/handler/export/PriorityQueue.java      |  218 +++
 .../org/apache/solr/handler/export/SortDoc.java |  114 ++
 .../apache/solr/handler/export/SortQueue.java   |   52 +
 .../apache/solr/handler/export/SortValue.java   |   30 +
 .../solr/handler/export/StringFieldWriter.java  |   52 +
 .../apache/solr/handler/export/StringValue.java |   99 +
 .../solr/handler/export/package-info.java       |   23 +
 .../collection1/conf/schema-sortingresponse.xml |    5 +-
 .../solr/handler/export/TestExportWriter.java   |  749 ++++++++
 .../apache/solr/response/TestExportWriter.java  |  370 ----
 .../src/exporting-result-sets.adoc              |    3 +-
 .../src/parallel-sql-interface.adoc             |    4 -
 .../org/apache/solr/common/SolrException.java   |    2 +-
 33 files changed, 2697 insertions(+), 2141 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9d9c3a0c/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 449f91d..e6d468c 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -169,6 +169,10 @@ Optimizations
 * SOLR-11654: Time Routed Alias will now route documents to the ideal shard of a collection, thus avoiding a hop.
   Usually documents were already routed well but not always.  (Gus Heck, David Smiley)
 
+* SOLR-11598: The export handler does not limit users to 4 sort fields and is now unlimited. However the speed at
+  which we can export is directly proportional to the number of sort fields specified. This change also allows streaming
+  expressions to group by on more than 4 fields. (Aroop Ganguly, Amrit Sarkar, Varun Thacker)
+
 Other Changes
 ----------------------
 

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9d9c3a0c/solr/core/src/java/org/apache/solr/handler/ExportHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/ExportHandler.java b/solr/core/src/java/org/apache/solr/handler/ExportHandler.java
index 9c75ef0..ea9239d 100644
--- a/solr/core/src/java/org/apache/solr/handler/ExportHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/ExportHandler.java
@@ -25,6 +25,7 @@ import org.apache.solr.common.params.CommonParams;
 import org.apache.solr.common.params.MapSolrParams;
 import org.apache.solr.common.params.SolrParams;
 import org.apache.solr.handler.component.SearchHandler;
+import org.apache.solr.handler.export.ExportWriter;
 import org.apache.solr.request.SolrQueryRequest;
 import org.apache.solr.response.SolrQueryResponse;
 

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9d9c3a0c/solr/core/src/java/org/apache/solr/handler/ExportWriter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/ExportWriter.java b/solr/core/src/java/org/apache/solr/handler/ExportWriter.java
deleted file mode 100644
index f618eef..0000000
--- a/solr/core/src/java/org/apache/solr/handler/ExportWriter.java
+++ /dev/null
@@ -1,1764 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.handler;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.io.OutputStreamWriter;
-import java.io.PrintWriter;
-import java.lang.invoke.MethodHandles;
-import java.nio.charset.StandardCharsets;
-import java.util.Date;
-import java.util.List;
-import java.util.function.LongFunction;
-
-import org.apache.lucene.index.DocValues;
-import org.apache.lucene.index.IndexableField;
-import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.MultiDocValues;
-import org.apache.lucene.index.NumericDocValues;
-import org.apache.lucene.index.OrdinalMap;
-import org.apache.lucene.index.SortedDocValues;
-import org.apache.lucene.index.SortedNumericDocValues;
-import org.apache.lucene.index.SortedSetDocValues;
-import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.search.Sort;
-import org.apache.lucene.search.SortField;
-import org.apache.lucene.util.ArrayUtil;
-import org.apache.lucene.util.BitSetIterator;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.CharsRefBuilder;
-import org.apache.lucene.util.FixedBitSet;
-import org.apache.lucene.util.LongValues;
-import org.apache.lucene.util.NumericUtils;
-import org.apache.solr.client.solrj.impl.BinaryResponseParser;
-import org.apache.solr.common.IteratorWriter;
-import org.apache.solr.common.MapWriter.EntryWriter;
-import org.apache.solr.common.MapWriter;
-import org.apache.solr.common.PushWriter;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.common.params.SolrParams;
-import org.apache.solr.common.util.JavaBinCodec;
-import org.apache.solr.core.SolrCore;
-import org.apache.solr.request.SolrQueryRequest;
-import org.apache.solr.request.SolrRequestInfo;
-import org.apache.solr.response.BinaryResponseWriter;
-import org.apache.solr.response.JSONResponseWriter;
-import org.apache.solr.response.QueryResponseWriter;
-import org.apache.solr.response.SolrQueryResponse;
-import org.apache.solr.schema.BoolField;
-import org.apache.solr.schema.DateValueFieldType;
-import org.apache.solr.schema.DoubleValueFieldType;
-import org.apache.solr.schema.FieldType;
-import org.apache.solr.schema.FloatValueFieldType;
-import org.apache.solr.schema.IndexSchema;
-import org.apache.solr.schema.IntValueFieldType;
-import org.apache.solr.schema.LongValueFieldType;
-import org.apache.solr.schema.SchemaField;
-import org.apache.solr.schema.StrField;
-import org.apache.solr.search.SolrIndexSearcher;
-import org.apache.solr.search.SortSpec;
-import org.apache.solr.search.SyntaxError;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static java.util.Collections.singletonList;
-import static java.util.Collections.singletonMap;
-import static org.apache.solr.common.util.Utils.makeMap;
-
-public class ExportWriter implements SolrCore.RawWriter, Closeable {
-  private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-  private OutputStreamWriter respWriter;
-  final SolrQueryRequest req;
-  final SolrQueryResponse res;
-  FieldWriter[] fieldWriters;
-  int totalHits = 0;
-  FixedBitSet[] sets = null;
-  PushWriter writer;
-  private String wt;
-
-
-  ExportWriter(SolrQueryRequest req, SolrQueryResponse res, String wt) {
-    this.req = req;
-    this.res = res;
-    this.wt = wt;
-
-  }
-
-  @Override
-  public String getContentType() {
-    if ("javabin".equals(wt)) {
-      return BinaryResponseParser.BINARY_CONTENT_TYPE;
-    } else return "json";
-  }
-
-  @Override
-  public void close() throws IOException {
-    if (writer != null) writer.close();
-    if (respWriter != null) {
-      respWriter.flush();
-      respWriter.close();
-    }
-
-  }
-
-  protected void writeException(Exception e, PushWriter w, boolean log) throws IOException {
-    w.writeMap(mw -> {
-      mw.put("responseHeader", singletonMap("status", 400))
-          .put("response", makeMap(
-              "numFound", 0,
-              "docs", singletonList(singletonMap("EXCEPTION", e.getMessage()))));
-    });
-    if (log) {
-      SolrException.log(logger, e);
-    }
-  }
-
-  public void write(OutputStream os) throws IOException {
-    QueryResponseWriter rw = req.getCore().getResponseWriters().get(wt);
-    if (rw instanceof BinaryResponseWriter) {
-      //todo add support for other writers after testing
-      writer = new JavaBinCodec(os, null);
-    } else {
-      respWriter = new OutputStreamWriter(os, StandardCharsets.UTF_8);
-      writer = JSONResponseWriter.getPushWriter(respWriter, req, res);
-    }
-    Exception exception = res.getException();
-    if (exception != null) {
-      if (!(exception instanceof IgnoreException)) {
-        writeException(exception, writer, false);
-      }
-      return;
-    }
-
-    SolrRequestInfo info = SolrRequestInfo.getRequestInfo();
-    SortSpec sortSpec = info.getResponseBuilder().getSortSpec();
-
-    if(sortSpec == null) {
-      writeException((new IOException(new SyntaxError("No sort criteria was provided."))), writer, true);
-      return;
-    }
-
-    SolrIndexSearcher searcher = req.getSearcher();
-    Sort sort = searcher.weightSort(sortSpec.getSort());
-
-    if(sort == null) {
-      writeException((new IOException(new SyntaxError("No sort criteria was provided."))), writer, true);
-      return;
-    }
-
-    if(sort != null && sort.needsScores()) {
-      writeException((new IOException(new SyntaxError("Scoring is not currently supported with xsort."))), writer, true);
-      return;
-    }
-
-    // There is a bailout in SolrIndexSearcher.getDocListNC when there are _no_ docs in the index at all.
-    // if (lastDocRequested <= 0) {
-    // That causes the totalHits and export entries in the context to _not_ get set.
-    // The only time that really matters is when we search against an _empty_ set. That's too obscure
-    // a condition to handle as part of this patch, if someone wants to pursue it it can be reproduced with:
-    // ant test  -Dtestcase=StreamingTest -Dtests.method=testAllValidExportTypes -Dtests.seed=10F13879D0D1D6AD -Dtests.slow=true -Dtests.locale=es-PA -Dtests.timezone=America/Bahia_Banderas -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
-    // You'll have to uncomment the if below to hit the null pointer exception.
-    // This is such an unusual case (i.e. an empty index) that catching this concdition here is probably OK.
-    // This came to light in the very artifical case of indexing a single doc to Cloud.
-    if (req.getContext().get("totalHits") != null) {
-      totalHits = ((Integer)req.getContext().get("totalHits")).intValue();
-      sets = (FixedBitSet[]) req.getContext().get("export");
-      if (sets == null) {
-        writeException((new IOException(new SyntaxError("xport RankQuery is required for xsort: rq={!xport}"))), writer, true);
-        return;
-      }
-    }
-    SolrParams params = req.getParams();
-    String fl = params.get("fl");
-
-    String[] fields = null;
-
-    if(fl == null) {
-      writeException((new IOException(new SyntaxError("export field list (fl) must be specified."))), writer, true);
-      return;
-    } else  {
-      fields = fl.split(",");
-
-      for(int i=0;i<fields.length; i++) {
-
-        fields[i] = fields[i].trim();
-
-        if(fields[i].equals("score")) {
-          writeException((new IOException(new SyntaxError("Scoring is not currently supported with xsort."))), writer, true);
-          return;
-        }
-      }
-    }
-
-    try {
-      fieldWriters = getFieldWriters(fields, req.getSearcher());
-    } catch (Exception e) {
-      writeException(e, writer, true);
-      return;
-    }
-
-    writer.writeMap(m -> {
-      m.put("responseHeader", singletonMap("status", 0));
-      m.put("response", (MapWriter) mw -> {
-        mw.put("numFound", totalHits);
-        mw.put("docs", (IteratorWriter) iw -> writeDocs(req, iw, sort));
-      });
-    });
-
-  }
-
-  protected void writeDocs(SolrQueryRequest req, IteratorWriter.ItemWriter writer, Sort sort) throws IOException {
-    //Write the data.
-    List<LeafReaderContext> leaves = req.getSearcher().getTopReaderContext().leaves();
-    SortDoc sortDoc = getSortDoc(req.getSearcher(), sort.getSort());
-    int count = 0;
-    int queueSize = 30000;
-    SortQueue queue = new SortQueue(queueSize, sortDoc);
-    SortDoc[] outDocs = new SortDoc[queueSize];
-
-    while(count < totalHits) {
-      //long begin = System.nanoTime();
-      queue.reset();
-      SortDoc top = queue.top();
-      for(int i=0; i<leaves.size(); i++) {
-        sortDoc.setNextReader(leaves.get(i));
-        DocIdSetIterator it = new BitSetIterator(sets[i], 0); // cost is not useful here
-        int docId = -1;
-        while((docId = it.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
-          sortDoc.setValues(docId);
-          if(top.lessThan(sortDoc)) {
-            top.setValues(sortDoc);
-            top = queue.updateTop();
-          }
-        }
-      }
-
-      int outDocsIndex = -1;
-
-      for(int i=0; i<queueSize; i++) {
-        SortDoc s = queue.pop();
-        if(s.docId > -1) {
-          outDocs[++outDocsIndex] = s;
-        }
-      }
-
-      //long end = System.nanoTime();
-
-      count += (outDocsIndex+1);
-
-      try {
-        for(int i=outDocsIndex; i>=0; --i) {
-          SortDoc s = outDocs[i];
-          writer.add((MapWriter) ew -> {
-            writeDoc(s, leaves, ew);
-            s.reset();
-          });
-        }
-      } catch(Throwable e) {
-        Throwable ex = e;
-        while(ex != null) {
-          String m = ex.getMessage();
-          if(m != null && m.contains("Broken pipe")) {
-            throw new IgnoreException();
-          }
-          ex = ex.getCause();
-        }
-
-        if(e instanceof IOException) {
-          throw ((IOException)e);
-        } else {
-          throw new IOException(e);
-        }
-      }
-    }
-  }
-
-  protected void writeDoc(SortDoc sortDoc,
-                          List<LeafReaderContext> leaves,
-                          EntryWriter ew) throws IOException {
-
-    int ord = sortDoc.ord;
-    FixedBitSet set = sets[ord];
-    set.clear(sortDoc.docId);
-    LeafReaderContext context = leaves.get(ord);
-    int fieldIndex = 0;
-    for (FieldWriter fieldWriter : fieldWriters) {
-      if (fieldWriter.write(sortDoc.docId, context.reader(), ew, fieldIndex)) {
-        ++fieldIndex;
-      }
-    }
-  }
-
-  protected FieldWriter[] getFieldWriters(String[] fields, SolrIndexSearcher searcher) throws IOException {
-    IndexSchema schema = searcher.getSchema();
-    FieldWriter[] writers = new FieldWriter[fields.length];
-    for(int i=0; i<fields.length; i++) {
-      String field = fields[i];
-      SchemaField schemaField = null;
-
-      try {
-        schemaField = schema.getField(field);
-      } catch (Exception e) {
-        throw new IOException(e);
-      }
-
-      if(!schemaField.hasDocValues()) {
-        throw new IOException(field+" must have DocValues to use this feature.");
-      }
-
-      boolean multiValued = schemaField.multiValued();
-      FieldType fieldType = schemaField.getType();
-      if (fieldType instanceof IntValueFieldType) {
-        if (multiValued) {
-          writers[i] = new MultiFieldWriter(field, fieldType, schemaField, true);
-        } else {
-          writers[i] = new IntFieldWriter(field);
-        }
-      } else if (fieldType instanceof LongValueFieldType) {
-        if (multiValued) {
-          writers[i] = new MultiFieldWriter(field, fieldType, schemaField, true);
-        } else {
-          writers[i] = new LongFieldWriter(field);
-        }
-      } else if (fieldType instanceof FloatValueFieldType) {
-        if (multiValued) {
-          writers[i] = new MultiFieldWriter(field, fieldType, schemaField, true);
-        } else {
-          writers[i] = new FloatFieldWriter(field);
-        }
-      } else if (fieldType instanceof DoubleValueFieldType) {
-        if (multiValued) {
-          writers[i] = new MultiFieldWriter(field, fieldType, schemaField, true);
-        } else {
-          writers[i] = new DoubleFieldWriter(field);
-        }
-      } else if (fieldType instanceof StrField) {
-        if (multiValued) {
-          writers[i] = new MultiFieldWriter(field, fieldType, schemaField, false);
-        } else {
-          writers[i] = new StringFieldWriter(field, fieldType);
-        }
-      } else if (fieldType instanceof DateValueFieldType) {
-        if (multiValued) {
-          writers[i] = new MultiFieldWriter(field, fieldType, schemaField, false);
-        } else {
-          writers[i] = new DateFieldWriter(field);
-        }
-      } else if (fieldType instanceof BoolField) {
-        if (multiValued) {
-          writers[i] = new MultiFieldWriter(field, fieldType, schemaField, true);
-        } else {
-          writers[i] = new BoolFieldWriter(field, fieldType);
-        }
-      } else {
-        throw new IOException("Export fields must either be one of the following types: int,float,long,double,string,date,boolean");
-      }
-    }
-    return writers;
-  }
-
-  private SortDoc getSortDoc(SolrIndexSearcher searcher, SortField[] sortFields) throws IOException {
-    SortValue[] sortValues = new SortValue[sortFields.length];
-    IndexSchema schema = searcher.getSchema();
-    for(int i=0; i<sortFields.length; ++i) {
-      SortField sf = sortFields[i];
-      String field = sf.getField();
-      boolean reverse = sf.getReverse();
-      SchemaField schemaField = schema.getField(field);
-      FieldType ft = schemaField.getType();
-
-      if(!schemaField.hasDocValues()) {
-        throw new IOException(field+" must have DocValues to use this feature.");
-      }
-
-      if(ft instanceof IntValueFieldType) {
-        if(reverse) {
-          sortValues[i] = new IntValue(field, new IntDesc());
-        } else {
-          sortValues[i] = new IntValue(field, new IntAsc());
-        }
-      } else if(ft instanceof FloatValueFieldType) {
-        if(reverse) {
-          sortValues[i] = new FloatValue(field, new FloatDesc());
-        } else {
-          sortValues[i] = new FloatValue(field, new FloatAsc());
-        }
-      } else if(ft instanceof DoubleValueFieldType) {
-        if(reverse) {
-          sortValues[i] = new DoubleValue(field, new DoubleDesc());
-        } else {
-          sortValues[i] = new DoubleValue(field, new DoubleAsc());
-        }
-      } else if(ft instanceof LongValueFieldType) {
-        if(reverse) {
-          sortValues[i] = new LongValue(field, new LongDesc());
-        } else {
-          sortValues[i] = new LongValue(field, new LongAsc());
-        }
-      } else if(ft instanceof StrField) {
-        LeafReader reader = searcher.getSlowAtomicReader();
-        SortedDocValues vals =  reader.getSortedDocValues(field);
-        if(reverse) {
-          sortValues[i] = new StringValue(vals, field, new IntDesc());
-        } else {
-          sortValues[i] = new StringValue(vals, field, new IntAsc());
-        }
-      } else if (ft instanceof DateValueFieldType) {
-        if (reverse) {
-          sortValues[i] = new LongValue(field, new LongDesc());
-        } else {
-          sortValues[i] = new LongValue(field, new LongAsc());
-        }
-      } else if (ft instanceof BoolField) {
-        // This is a bit of a hack, but since the boolean field stores ByteRefs, just like Strings
-        // _and_ since "F" happens to sort before "T" (thus false sorts "less" than true)
-        // we can just use the existing StringValue here.
-        LeafReader reader = searcher.getSlowAtomicReader();
-        SortedDocValues vals = reader.getSortedDocValues(field);
-        if (reverse) {
-          sortValues[i] = new StringValue(vals, field, new IntDesc());
-        } else {
-          sortValues[i] = new StringValue(vals, field, new IntAsc());
-        }
-      } else {
-        throw new IOException("Sort fields must be one of the following types: int,float,long,double,string,date,boolean");
-      }
-    }
-
-    if(sortValues.length == 1) {
-      return new SingleValueSortDoc(sortValues[0]);
-    } else if(sortValues.length == 2) {
-      return new DoubleValueSortDoc(sortValues[0], sortValues[1]);
-    } else if(sortValues.length == 3) {
-      return new TripleValueSortDoc(sortValues[0], sortValues[1], sortValues[2]);
-    } else if(sortValues.length == 4) {
-      return new QuadValueSortDoc(sortValues[0], sortValues[1], sortValues[2], sortValues[3]);
-    } else {
-      throw new IOException("A max of 4 sorts can be specified");
-    }
-  }
-
-  class SortQueue extends PriorityQueue<SortDoc> {
-
-    private SortDoc proto;
-    private Object[] cache;
-
-    public SortQueue(int len, SortDoc proto) {
-      super(len);
-      this.proto = proto;
-    }
-
-    protected boolean lessThan(SortDoc t1, SortDoc t2) {
-      return t1.lessThan(t2);
-    }
-
-    private void populate() {
-      Object[] heap = getHeapArray();
-      cache = new SortDoc[heap.length];
-      for (int i = 1; i < heap.length; i++) {
-        cache[i] = heap[i] = proto.copy();
-      }
-      size = maxSize;
-    }
-
-    private void reset() {
-      Object[] heap = getHeapArray();
-      if(cache != null) {
-        System.arraycopy(cache, 1, heap, 1, heap.length-1);
-        size = maxSize;
-      } else {
-        populate();
-      }
-    }
-  }
-
-  class SortDoc {
-
-    protected int docId = -1;
-    protected int ord = -1;
-    protected int docBase = -1;
-
-    private SortValue[] sortValues;
-
-    public SortDoc() {
-
-    }
-
-    public void setNextReader(LeafReaderContext context) throws IOException {
-      this.ord = context.ord;
-      for (SortValue value : sortValues) {
-        value.setNextReader(context);
-      }
-    }
-
-    public void reset() {
-      this.docId = -1;
-    }
-
-    public void setValues(int docId) throws IOException {
-      this.docId = docId;
-      for(SortValue sortValue : sortValues) {
-        sortValue.setCurrentValue(docId);
-      }
-    }
-
-    public void setValues(SortDoc sortDoc) throws IOException {
-      this.docId = sortDoc.docId;
-      this.ord = sortDoc.ord;
-      SortValue[] vals = sortDoc.sortValues;
-      for(int i=0; i<vals.length; i++) {
-        sortValues[i].setCurrentValue(vals[i]);
-      }
-    }
-
-    public SortDoc(SortValue[] sortValues) {
-      this.sortValues = sortValues;
-    }
-
-    public SortDoc copy() {
-      SortValue[] svs = new SortValue[sortValues.length];
-      for(int i=0; i<sortValues.length; i++) {
-        svs[i] = sortValues[i].copy();
-      }
-
-      return new SortDoc(svs);
-    }
-
-    public boolean lessThan(Object o) {
-      if(docId == -1) {
-        return true;
-      }
-
-      SortDoc sd = (SortDoc)o;
-      SortValue[] sortValues1 = sd.sortValues;
-      for(int i=0; i<sortValues.length; i++) {
-        int comp = sortValues[i].compareTo(sortValues1[i]);
-        if(comp < 0) {
-          return true;
-        } if(comp > 0) {
-          return false;
-        }
-      }
-      return docId+docBase < sd.docId+sd.docBase;
-    }
-
-    public String toString() {
-      return "";
-    }
-  }
-
-  class SingleValueSortDoc extends SortDoc {
-
-    protected SortValue value1;
-
-    public void setNextReader(LeafReaderContext context) throws IOException {
-      this.ord = context.ord;
-      value1.setNextReader(context);
-    }
-
-    public void reset() {
-      this.docId = -1;
-      this.value1.reset();
-    }
-
-    public void setValues(int docId) throws IOException {
-      this.docId = docId;
-      value1.setCurrentValue(docId);
-    }
-
-    public void setValues(SortDoc sortDoc) throws IOException {
-      this.docId = sortDoc.docId;
-      this.ord = sortDoc.ord;
-      value1.setCurrentValue(((SingleValueSortDoc)sortDoc).value1);
-    }
-
-    public SingleValueSortDoc(SortValue value1) {
-      super();
-      this.value1 = value1;
-    }
-
-    public SortDoc copy() {
-      return new SingleValueSortDoc(value1.copy());
-    }
-
-    public boolean lessThan(Object o) {
-      SingleValueSortDoc sd = (SingleValueSortDoc)o;
-      int comp = value1.compareTo(sd.value1);
-      if(comp == -1) {
-        return true;
-      } else if (comp == 1) {
-        return false;
-      } else {
-        return docId+docBase > sd.docId+sd.docBase;
-      }
-    }
-
-    public int compareTo(Object o) {
-      SingleValueSortDoc sd = (SingleValueSortDoc)o;
-      return value1.compareTo(sd.value1);
-    }
-
-    public String toString() {
-      return docId+":"+value1.toString();
-    }
-  }
-
-  class DoubleValueSortDoc extends SingleValueSortDoc {
-
-    protected SortValue value2;
-
-    public void setNextReader(LeafReaderContext context) throws IOException {
-      this.ord = context.ord;
-      value1.setNextReader(context);
-      value2.setNextReader(context);
-    }
-
-    public void reset() {
-      this.docId = -1;
-      value1.reset();
-      value2.reset();
-    }
-
-    public void setValues(int docId) throws IOException {
-      this.docId = docId;
-      value1.setCurrentValue(docId);
-      value2.setCurrentValue(docId);
-    }
-
-    public void setValues(SortDoc sortDoc) throws IOException {
-      this.docId = sortDoc.docId;
-      this.ord = sortDoc.ord;
-      value1.setCurrentValue(((DoubleValueSortDoc)sortDoc).value1);
-      value2.setCurrentValue(((DoubleValueSortDoc)sortDoc).value2);
-    }
-
-    public DoubleValueSortDoc(SortValue value1, SortValue value2) {
-      super(value1);
-      this.value2 = value2;
-    }
-
-    public SortDoc copy() {
-      return new DoubleValueSortDoc(value1.copy(), value2.copy());
-    }
-
-    public boolean lessThan(Object o) {
-      DoubleValueSortDoc sd = (DoubleValueSortDoc)o;
-      int comp = value1.compareTo(sd.value1);
-      if(comp == -1) {
-        return true;
-      } else if (comp == 1) {
-        return false;
-      } else {
-        comp = value2.compareTo(sd.value2);
-        if(comp == -1) {
-          return true;
-        } else if (comp == 1) {
-          return false;
-        } else {
-          return docId+docBase > sd.docId+sd.docBase;
-        }
-      }
-    }
-
-    public int compareTo(Object o) {
-      DoubleValueSortDoc sd = (DoubleValueSortDoc)o;
-      int comp = value1.compareTo(sd.value1);
-      if(comp == 0) {
-        return value2.compareTo(sd.value2);
-      } else {
-        return comp;
-      }
-    }
-  }
-
-  class TripleValueSortDoc extends DoubleValueSortDoc {
-
-    protected SortValue value3;
-
-    public void setNextReader(LeafReaderContext context) throws IOException {
-      this.ord = context.ord;
-      value1.setNextReader(context);
-      value2.setNextReader(context);
-      value3.setNextReader(context);
-    }
-
-    public void reset() {
-      this.docId = -1;
-      value1.reset();
-      value2.reset();
-      value3.reset();
-    }
-
-    public void setValues(int docId) throws IOException {
-      this.docId = docId;
-      value1.setCurrentValue(docId);
-      value2.setCurrentValue(docId);
-      value3.setCurrentValue(docId);
-    }
-
-    public void setValues(SortDoc sortDoc) throws IOException {
-      this.docId = sortDoc.docId;
-      this.ord = sortDoc.ord;
-      value1.setCurrentValue(((TripleValueSortDoc)sortDoc).value1);
-      value2.setCurrentValue(((TripleValueSortDoc)sortDoc).value2);
-      value3.setCurrentValue(((TripleValueSortDoc)sortDoc).value3);
-    }
-
-    public TripleValueSortDoc(SortValue value1, SortValue value2, SortValue value3) {
-      super(value1, value2);
-      this.value3 = value3;
-    }
-
-    public SortDoc copy() {
-      return new TripleValueSortDoc(value1.copy(), value2.copy(), value3.copy());
-    }
-
-    public boolean lessThan(Object o) {
-
-      TripleValueSortDoc sd = (TripleValueSortDoc)o;
-      int comp = value1.compareTo(sd.value1);
-      if(comp == -1) {
-        return true;
-      } else if (comp == 1) {
-        return false;
-      } else {
-        comp = value2.compareTo(sd.value2);
-        if(comp == -1) {
-          return true;
-        } else if (comp == 1) {
-          return false;
-        } else {
-          comp = value3.compareTo(sd.value3);
-          if(comp == -1) {
-            return true;
-          } else if (comp == 1) {
-            return false;
-          } else {
-            return docId+docBase > sd.docId+sd.docBase;
-          }
-        }
-      }
-    }
-
-    public int compareTo(Object o) {
-
-      TripleValueSortDoc sd = (TripleValueSortDoc)o;
-      int comp = value1.compareTo(sd.value1);
-      if(comp == 0) {
-        comp = value2.compareTo(sd.value2);
-        if(comp == 0) {
-          return value3.compareTo(sd.value3);
-        } else {
-          return comp;
-        }
-      } else {
-        return comp;
-      }
-    }
-  }
-
-  class QuadValueSortDoc extends TripleValueSortDoc {
-
-    protected SortValue value4;
-
-    public void setNextReader(LeafReaderContext context) throws IOException {
-      this.ord = context.ord;
-      value1.setNextReader(context);
-      value2.setNextReader(context);
-      value3.setNextReader(context);
-      value4.setNextReader(context);
-    }
-
-    public void reset() {
-      this.docId = -1;
-      value1.reset();
-      value2.reset();
-      value3.reset();
-      value4.reset();
-    }
-
-    public void setValues(int docId) throws IOException {
-      this.docId = docId;
-      value1.setCurrentValue(docId);
-      value2.setCurrentValue(docId);
-      value3.setCurrentValue(docId);
-      value4.setCurrentValue(docId);
-    }
-
-    public void setValues(SortDoc sortDoc) throws IOException {
-      this.docId = sortDoc.docId;
-      this.ord = sortDoc.ord;
-      value1.setCurrentValue(((QuadValueSortDoc)sortDoc).value1);
-      value2.setCurrentValue(((QuadValueSortDoc)sortDoc).value2);
-      value3.setCurrentValue(((QuadValueSortDoc)sortDoc).value3);
-      value4.setCurrentValue(((QuadValueSortDoc)sortDoc).value4);
-    }
-
-    public QuadValueSortDoc(SortValue value1, SortValue value2, SortValue value3, SortValue value4) {
-      super(value1, value2, value3);
-      this.value4 = value4;
-    }
-
-    public SortDoc copy() {
-      return new QuadValueSortDoc(value1.copy(), value2.copy(), value3.copy(), value4.copy());
-    }
-
-    public boolean lessThan(Object o) {
-
-      QuadValueSortDoc sd = (QuadValueSortDoc)o;
-      int comp = value1.compareTo(sd.value1);
-      if(comp == -1) {
-        return true;
-      } else if (comp == 1) {
-        return false;
-      } else {
-        comp = value2.compareTo(sd.value2);
-        if(comp == -1) {
-          return true;
-        } else if (comp == 1) {
-          return false;
-        } else {
-          comp = value3.compareTo(sd.value3);
-          if(comp == -1) {
-            return true;
-          } else if (comp == 1) {
-            return false;
-          } else {
-            comp = value4.compareTo(sd.value4);
-            if(comp == -1) {
-              return true;
-            } else if (comp == 1) {
-              return false;
-            } else {
-              return docId+docBase > sd.docId+sd.docBase;
-            }
-          }
-        }
-      }
-    }
-
-    public int compareTo(Object o) {
-      QuadValueSortDoc sd = (QuadValueSortDoc)o;
-      int comp = value1.compareTo(sd.value1);
-      if(comp == 0) {
-        comp = value2.compareTo(sd.value2);
-        if(comp == 0) {
-          comp = value3.compareTo(sd.value3);
-          if(comp == 0) {
-            return value4.compareTo(sd.value4);
-          } else {
-            return comp;
-          }
-        } else {
-          return comp;
-        }
-      } else {
-        return comp;
-      }
-    }
-  }
-
-  public interface SortValue extends Comparable<SortValue> {
-    public void setCurrentValue(int docId) throws IOException;
-    public void setNextReader(LeafReaderContext context) throws IOException;
-    public void setCurrentValue(SortValue value);
-    public void reset();
-    public SortValue copy();
-  }
-
-  class IntValue implements SortValue {
-
-    protected NumericDocValues vals;
-    protected String field;
-    protected int currentValue;
-    protected IntComp comp;
-    private int lastDocID;
-
-    public IntValue copy() {
-      return new IntValue(field, comp);
-    }
-
-    public IntValue(String field, IntComp comp) {
-      this.field = field;
-      this.comp = comp;
-      this.currentValue = comp.resetValue();
-    }
-
-    public void setNextReader(LeafReaderContext context) throws IOException {
-      this.vals = DocValues.getNumeric(context.reader(), field);
-      lastDocID = 0;
-    }
-
-    public void setCurrentValue(int docId) throws IOException {
-      if (docId < lastDocID) {
-        throw new AssertionError("docs were sent out-of-order: lastDocID=" + lastDocID + " vs doc=" + docId);
-      }
-      lastDocID = docId;
-      int curDocID = vals.docID();
-      if (docId > curDocID) {
-        curDocID = vals.advance(docId);
-      }
-      if (docId == curDocID) {
-        currentValue = (int) vals.longValue();
-      } else {
-        currentValue = 0;
-      }
-    }
-
-    public int compareTo(SortValue o) {
-      IntValue iv = (IntValue)o;
-      return comp.compare(currentValue, iv.currentValue);
-    }
-
-    public void setCurrentValue (SortValue value) {
-      currentValue = ((IntValue)value).currentValue;
-    }
-
-    public void reset() {
-      currentValue = comp.resetValue();
-    }
-  }
-
-  public interface IntComp {
-    public int compare(int a, int b);
-    public int resetValue();
-  }
-
-  static class IntDesc implements IntComp {
-
-    public int resetValue() {
-      return Integer.MIN_VALUE;
-    }
-
-    public int compare(int a, int b) {
-      if(a < b) {
-        return -1;
-      } else if (a > b) {
-        return 1;
-      } else {
-        return 0;
-      }
-    }
-  }
-
-  static class IntAsc implements IntComp {
-
-    public int resetValue() {
-      return Integer.MAX_VALUE;
-    }
-
-    public int compare(int a, int b) {
-      if(a < b) {
-        return 1;
-      } else if (a > b) {
-        return -1;
-      } else {
-        return 0;
-      }
-    }
-  }
-
-  class LongValue implements SortValue {
-
-    protected NumericDocValues vals;
-    protected String field;
-    protected long currentValue;
-    protected LongComp comp;
-    private int lastDocID;
-
-    public LongValue(String field, LongComp comp) {
-      this.field = field;
-      this.comp = comp;
-      this.currentValue = comp.resetValue();
-    }
-
-    public LongValue copy() {
-      return new LongValue(field, comp);
-    }
-
-    public void setNextReader(LeafReaderContext context) throws IOException {
-      this.vals = DocValues.getNumeric(context.reader(), field);
-      lastDocID = 0;
-    }
-
-    public void setCurrentValue(int docId) throws IOException {
-      if (docId < lastDocID) {
-        throw new AssertionError("docs were sent out-of-order: lastDocID=" + lastDocID + " vs doc=" + docId);
-      }
-      lastDocID = docId;
-      int curDocID = vals.docID();
-      if (docId > curDocID) {
-        curDocID = vals.advance(docId);
-      }
-      if (docId == curDocID) {
-        currentValue = vals.longValue();
-      } else {
-        currentValue = 0;
-      }
-    }
-
-    public void setCurrentValue(SortValue sv) {
-      LongValue lv = (LongValue)sv;
-      this.currentValue = lv.currentValue;
-    }
-
-    public int compareTo(SortValue o) {
-      LongValue l = (LongValue)o;
-      return comp.compare(currentValue, l.currentValue);
-    }
-
-    public void reset() {
-      this.currentValue = comp.resetValue();
-    }
-  }
-
-  interface LongComp {
-    public int compare(long a, long b);
-    public long resetValue();
-  }
-
-  static class LongDesc implements LongComp {
-
-    public long resetValue() {
-      return Long.MIN_VALUE;
-    }
-
-    public int compare(long a, long b) {
-      if(a < b) {
-        return -1;
-      } else if (a > b) {
-        return 1;
-      } else {
-        return 0;
-      }
-    }
-  }
-
-  static class LongAsc implements LongComp {
-
-    public long resetValue() {
-      return Long.MAX_VALUE;
-    }
-
-    public int compare(long a, long b) {
-      if(a < b) {
-        return 1;
-      } else if (a > b) {
-        return -1;
-      } else {
-        return 0;
-      }
-    }
-  }
-
-  class FloatValue implements SortValue {
-
-    protected NumericDocValues vals;
-    protected String field;
-    protected float currentValue;
-    protected FloatComp comp;
-    private int lastDocID;
-
-    public FloatValue(String field, FloatComp comp) {
-      this.field = field;
-      this.comp = comp;
-      this.currentValue = comp.resetValue();
-    }
-
-    public FloatValue copy() {
-      return new FloatValue(field, comp);
-    }
-
-    public void setNextReader(LeafReaderContext context) throws IOException {
-      this.vals = DocValues.getNumeric(context.reader(), field);
-      lastDocID = 0;
-    }
-
-    public void setCurrentValue(int docId) throws IOException {
-      if (docId < lastDocID) {
-        throw new AssertionError("docs were sent out-of-order: lastDocID=" + lastDocID + " vs doc=" + docId);
-      }
-      lastDocID = docId;
-      int curDocID = vals.docID();
-      if (docId > curDocID) {
-        curDocID = vals.advance(docId);
-      }
-      if (docId == curDocID) {
-        currentValue = Float.intBitsToFloat((int)vals.longValue());
-      } else {
-        currentValue = 0f;
-      }
-    }
-
-    public void setCurrentValue(SortValue sv) {
-      FloatValue fv = (FloatValue)sv;
-      this.currentValue = fv.currentValue;
-    }
-
-    public void reset() {
-      this.currentValue = comp.resetValue();
-    }
-
-    public int compareTo(SortValue o) {
-      FloatValue fv = (FloatValue)o;
-      return comp.compare(currentValue, fv.currentValue);
-    }
-  }
-
-  interface FloatComp {
-    public int compare(float a, float b);
-    public float resetValue();
-  }
-
-  public static class FloatDesc implements FloatComp {
-    public float resetValue() {
-      return -Float.MAX_VALUE;
-    }
-
-    public int compare(float a, float b) {
-      if(a < b) {
-        return -1;
-      } else if (a > b) {
-        return 1;
-      } else {
-        return 0;
-      }
-    }
-  }
-
-  public static class FloatAsc implements FloatComp {
-    public float resetValue() {
-      return Float.MAX_VALUE;
-    }
-
-    public int compare(float a, float b) {
-      if(a < b) {
-        return 1;
-      } else if (a > b) {
-        return -1;
-      } else {
-        return 0;
-      }
-    }
-  }
-
-  class DoubleValue implements SortValue {
-
-    protected NumericDocValues vals;
-    protected String field;
-    protected double currentValue;
-    protected DoubleComp comp;
-    private int lastDocID;
-    private LeafReader reader;
-
-    public DoubleValue(String field, DoubleComp comp) {
-      this.field = field;
-      this.comp = comp;
-      this.currentValue = comp.resetValue();
-    }
-
-    public DoubleValue copy() {
-      return new DoubleValue(field, comp);
-    }
-
-    public void setNextReader(LeafReaderContext context) throws IOException {
-      this.reader = context.reader();
-      this.vals = DocValues.getNumeric(this.reader, this.field);
-      lastDocID = 0;
-    }
-
-    public void setCurrentValue(int docId) throws IOException {
-      if (docId < lastDocID) {
-        // TODO: can we enforce caller to go in order instead?
-        this.vals = DocValues.getNumeric(this.reader, this.field);
-      }
-      lastDocID = docId;
-      int curDocID = vals.docID();
-      if (docId > curDocID) {
-        curDocID = vals.advance(docId);
-      }
-      if (docId == curDocID) {
-        currentValue = Double.longBitsToDouble(vals.longValue());
-      } else {
-        currentValue = 0f;
-      }
-    }
-
-    public void setCurrentValue(SortValue sv) {
-      DoubleValue dv = (DoubleValue)sv;
-      this.currentValue = dv.currentValue;
-    }
-
-    public void reset() {
-      this.currentValue = comp.resetValue();
-    }
-
-    public int compareTo(SortValue o) {
-      DoubleValue dv = (DoubleValue)o;
-      return comp.compare(currentValue, dv.currentValue);
-    }
-  }
-
-  interface DoubleComp {
-    public int compare(double a, double b);
-    public double resetValue();
-  }
-
-  public static class DoubleDesc implements DoubleComp {
-    public double resetValue() {
-      return -Double.MAX_VALUE;
-    }
-
-    public int compare(double a, double b) {
-      if(a < b) {
-        return -1;
-      } else if (a > b) {
-        return 1;
-      } else {
-        return 0;
-      }
-    }
-  }
-
-  public static class DoubleAsc implements DoubleComp {
-    public double resetValue() {
-      return Double.MAX_VALUE;
-    }
-
-    public int compare(double a, double b) {
-      if(a < b) {
-        return 1;
-      } else if (a > b) {
-        return -1;
-      } else {
-        return 0;
-      }
-    }
-  }
-
-  class StringValue implements SortValue {
-
-    protected SortedDocValues vals;
-
-    protected OrdinalMap ordinalMap;
-    protected LongValues globalOrds;
-    protected SortedDocValues currentVals;
-
-    protected String field;
-    protected int segment;
-    protected int currentOrd;
-    protected IntComp comp;
-    protected int lastDocID;
-
-    public StringValue(SortedDocValues vals, String field, IntComp comp)  {
-      this.vals = vals;
-      if(vals instanceof MultiDocValues.MultiSortedDocValues) {
-        this.ordinalMap = ((MultiDocValues.MultiSortedDocValues) vals).mapping;
-      }
-      this.field = field;
-      this.comp = comp;
-      this.currentOrd = comp.resetValue();
-    }
-
-    public StringValue copy() {
-      return new StringValue(vals, field, comp);
-    }
-
-    public void setCurrentValue(int docId) throws IOException {
-
-      if (docId < lastDocID) {
-        throw new AssertionError("docs were sent out-of-order: lastDocID=" + lastDocID + " vs doc=" + docId);
-      }
-
-      lastDocID = docId;
-
-      if (docId > currentVals.docID()) {
-        currentVals.advance(docId);
-      }
-      if (docId == currentVals.docID()) {
-        int ord = currentVals.ordValue();
-        if(globalOrds != null) {
-          currentOrd = (int)globalOrds.get(ord);
-        } else {
-          currentOrd = ord;
-        }
-      } else {
-        currentOrd = -1;
-      }
-    }
-
-    public void setCurrentValue(SortValue sv) {
-      StringValue v = (StringValue)sv;
-      this.currentOrd = v.currentOrd;
-    }
-
-    public void setNextReader(LeafReaderContext context) throws IOException {
-      segment = context.ord;
-      if(ordinalMap != null) {
-        globalOrds = ordinalMap.getGlobalOrds(segment);
-      }
-      currentVals = DocValues.getSorted(context.reader(), field);
-      lastDocID = 0;
-    }
-
-    public void reset() {
-      this.currentOrd = comp.resetValue();
-    }
-
-    public int compareTo(SortValue o) {
-      StringValue sv = (StringValue)o;
-      return comp.compare(currentOrd, sv.currentOrd);
-    }
-
-    public String toString() {
-      return Integer.toString(this.currentOrd);
-    }
-  }
-
-  protected abstract class FieldWriter {
-    public abstract boolean write(int docId, LeafReader reader, EntryWriter out, int fieldIndex) throws IOException;
-  }
-
-  class IntFieldWriter extends FieldWriter {
-    private String field;
-
-    public IntFieldWriter(String field) {
-      this.field = field;
-    }
-
-    public boolean write(int docId, LeafReader reader, EntryWriter ew, int fieldIndex) throws IOException {
-      NumericDocValues vals = DocValues.getNumeric(reader, this.field);
-      int val;
-      if (vals.advance(docId) == docId) {
-        val = (int) vals.longValue();
-      } else {
-        return false;
-      }
-      ew.put(this.field, val);
-      return true;
-    }
-  }
-  
-  static LongFunction<Object> bitsToValue(FieldType fieldType) {
-    switch (fieldType.getNumberType()) {
-      case LONG:
-        return (bits)-> bits;
-      case DATE:
-        return (bits)-> new Date(bits);
-      case INTEGER:
-        return (bits)-> (int)bits;
-      case FLOAT:
-        return (bits)-> NumericUtils.sortableIntToFloat((int)bits);
-      case DOUBLE:
-        return (bits)-> NumericUtils.sortableLongToDouble(bits);
-      default:
-        throw new AssertionError("Unsupported NumberType: " + fieldType.getNumberType());
-    }
-  }
-
-  class MultiFieldWriter extends FieldWriter {
-    private String field;
-    private FieldType fieldType;
-    private SchemaField schemaField;
-    private boolean numeric;
-    private CharsRefBuilder cref = new CharsRefBuilder();
-    private final LongFunction<Object> bitsToValue;
-
-    public MultiFieldWriter(String field, FieldType fieldType, SchemaField schemaField, boolean numeric) {
-      this.field = field;
-      this.fieldType = fieldType;
-      this.schemaField = schemaField;
-      this.numeric = numeric;
-      if (this.fieldType.isPointField()) {
-        bitsToValue = bitsToValue(fieldType);
-      } else {
-        bitsToValue = null;
-      }
-    }
-
-    public boolean write(int docId, LeafReader reader, EntryWriter out, int fieldIndex) throws IOException {
-      if (this.fieldType.isPointField()) {
-        SortedNumericDocValues vals = DocValues.getSortedNumeric(reader, this.field);
-        if (!vals.advanceExact(docId)) return false;
-        out.put(this.field,
-            (IteratorWriter) w -> {
-              for (int i = 0; i < vals.docValueCount(); i++) {
-                w.add(bitsToValue.apply(vals.nextValue()));
-              }
-            });
-        return true;
-      } else {
-        SortedSetDocValues vals = DocValues.getSortedSet(reader, this.field);
-        if (vals.advance(docId) != docId) return false;
-        out.put(this.field,
-            (IteratorWriter) w -> {
-              long o;
-              while((o = vals.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
-                BytesRef ref = vals.lookupOrd(o);
-                fieldType.indexedToReadable(ref, cref);
-                IndexableField f = fieldType.createField(schemaField, cref.toString());
-                if (f == null) w.add(cref.toString());
-                else w.add(fieldType.toObject(f));
-              }
-            });
-        return true;
-      }
-      
-    }
-  }
-
-  class LongFieldWriter extends FieldWriter {
-    private String field;
-
-    public LongFieldWriter(String field) {
-      this.field = field;
-    }
-
-    public boolean write(int docId, LeafReader reader, EntryWriter ew, int fieldIndex) throws IOException {
-      NumericDocValues vals = DocValues.getNumeric(reader, this.field);
-      long val;
-      if (vals.advance(docId) == docId) {
-        val = vals.longValue();
-      } else {
-        return false;
-      }
-      ew.put(field, val);
-      return true;
-    }
-  }
-
-  class DateFieldWriter extends FieldWriter {
-    private String field;
-
-    public DateFieldWriter(String field) {
-      this.field = field;
-    }
-
-    public boolean write(int docId, LeafReader reader, EntryWriter ew, int fieldIndex) throws IOException {
-      NumericDocValues vals = DocValues.getNumeric(reader, this.field);
-      long val;
-      if (vals.advance(docId) == docId) {
-        val = vals.longValue();
-      } else {
-        return false;
-      }
-      ew.put(this.field, new Date(val));
-      return true;
-    }
-  }
-
-  class BoolFieldWriter extends FieldWriter {
-    private String field;
-    private FieldType fieldType;
-    private CharsRefBuilder cref = new CharsRefBuilder();
-
-    public BoolFieldWriter(String field, FieldType fieldType) {
-      this.field = field;
-      this.fieldType = fieldType;
-    }
-
-    public boolean write(int docId, LeafReader reader, EntryWriter ew, int fieldIndex) throws IOException {
-      SortedDocValues vals = DocValues.getSorted(reader, this.field);
-      if (vals.advance(docId) != docId) {
-        return false;
-      }
-      int ord = vals.ordValue();
-
-      BytesRef ref = vals.lookupOrd(ord);
-      fieldType.indexedToReadable(ref, cref);
-      ew.put(this.field, "true".equals(cref.toString()));
-      return true;
-    }
-  }
-
-  class FloatFieldWriter extends FieldWriter {
-    private String field;
-
-    public FloatFieldWriter(String field) {
-      this.field = field;
-    }
-
-    public boolean write(int docId, LeafReader reader, EntryWriter ew, int fieldIndex) throws IOException {
-      NumericDocValues vals = DocValues.getNumeric(reader, this.field);
-      int val;
-      if (vals.advance(docId) == docId) {
-        val = (int)vals.longValue();
-      } else {
-        return false;
-      }
-      ew.put(this.field, Float.intBitsToFloat(val));
-      return true;
-    }
-  }
-
-  class DoubleFieldWriter extends FieldWriter {
-    private String field;
-
-    public DoubleFieldWriter(String field) {
-      this.field = field;
-    }
-
-    public boolean write(int docId, LeafReader reader, EntryWriter ew, int fieldIndex) throws IOException {
-      NumericDocValues vals = DocValues.getNumeric(reader, this.field);
-      long val;
-      if (vals.advance(docId) == docId) {
-        val = vals.longValue();
-      } else {
-        return false;
-      }
-      ew.put(this.field, Double.longBitsToDouble(val));
-      return true;
-    }
-  }
-
-  class StringFieldWriter extends FieldWriter {
-    private String field;
-    private FieldType fieldType;
-    private CharsRefBuilder cref = new CharsRefBuilder();
-
-    public StringFieldWriter(String field, FieldType fieldType) {
-      this.field = field;
-      this.fieldType = fieldType;
-    }
-
-    public boolean write(int docId, LeafReader reader, EntryWriter ew, int fieldIndex) throws IOException {
-      SortedDocValues vals = DocValues.getSorted(reader, this.field);
-      if (vals.advance(docId) != docId) {
-        return false;
-      }
-      int ord = vals.ordValue();
-
-      BytesRef ref = vals.lookupOrd(ord);
-      fieldType.indexedToReadable(ref, cref);
-      ew.put(this.field, cref.toString());
-      return true;
-    }
-  }
-
-  public abstract class PriorityQueue<T> {
-    protected int size = 0;
-    protected final int maxSize;
-    private final T[] heap;
-
-    public PriorityQueue(int maxSize) {
-      this(maxSize, true);
-    }
-
-    public PriorityQueue(int maxSize, boolean prepopulate) {
-      final int heapSize;
-      if (0 == maxSize) {
-        // We allocate 1 extra to avoid if statement in top()
-        heapSize = 2;
-      } else {
-        if (maxSize > ArrayUtil.MAX_ARRAY_LENGTH) {
-          // Don't wrap heapSize to -1, in this case, which
-          // causes a confusing NegativeArraySizeException.
-          // Note that very likely this will simply then hit
-          // an OOME, but at least that's more indicative to
-          // caller that this values is too big.  We don't +1
-          // in this case, but it's very unlikely in practice
-          // one will actually insert this many objects into
-          // the PQ:
-          // Throw exception to prevent confusing OOME:
-          throw new IllegalArgumentException("maxSize must be <= " + ArrayUtil.MAX_ARRAY_LENGTH + "; got: " + maxSize);
-        } else {
-          // NOTE: we add +1 because all access to heap is
-          // 1-based not 0-based.  heap[0] is unused.
-          heapSize = maxSize + 1;
-        }
-      }
-      // T is unbounded type, so this unchecked cast works always:
-      @SuppressWarnings("unchecked") final T[] h = (T[]) new Object[heapSize];
-      this.heap = h;
-      this.maxSize = maxSize;
-
-      if (prepopulate) {
-        // If sentinel objects are supported, populate the queue with them
-        T sentinel = getSentinelObject();
-        if (sentinel != null) {
-          heap[1] = sentinel;
-          for (int i = 2; i < heap.length; i++) {
-            heap[i] = getSentinelObject();
-          }
-          size = maxSize;
-        }
-      }
-    }
-
-    /** Determines the ordering of objects in this priority queue.  Subclasses
-     *  must define this one method.
-     *  @return <code>true</code> iff parameter <tt>a</tt> is less than parameter <tt>b</tt>.
-     */
-    protected abstract boolean lessThan(T a, T b);
-
-
-    protected T getSentinelObject() {
-      return null;
-    }
-
-    /**
-     * Adds an Object to a PriorityQueue in log(size) time. If one tries to add
-     * more objects than maxSize from initialize an
-     *
-     * @return the new 'top' element in the queue.
-     */
-    public final T add(T element) {
-      size++;
-      heap[size] = element;
-      upHeap();
-      return heap[1];
-    }
-
-    /**
-     * Adds an Object to a PriorityQueue in log(size) time.
-     * It returns the object (if any) that was
-     * dropped off the heap because it was full. This can be
-     * the given parameter (in case it is smaller than the
-     * full heap's minimum, and couldn't be added), or another
-     * object that was previously the smallest value in the
-     * heap and now has been replaced by a larger one, or null
-     * if the queue wasn't yet full with maxSize elements.
-     */
-    public T insertWithOverflow(T element) {
-      if (size < maxSize) {
-        add(element);
-        return null;
-      } else if (size > 0 && !lessThan(element, heap[1])) {
-        T ret = heap[1];
-        heap[1] = element;
-        updateTop();
-        return ret;
-      } else {
-        return element;
-      }
-    }
-
-    /** Returns the least element of the PriorityQueue in constant time. */
-    public final T top() {
-      // We don't need to check size here: if maxSize is 0,
-      // then heap is length 2 array with both entries null.
-      // If size is 0 then heap[1] is already null.
-      return heap[1];
-    }
-
-    /** Removes and returns the least element of the PriorityQueue in log(size)
-     time. */
-    public final T pop() {
-      if (size > 0) {
-        T result = heap[1];       // save first value
-        heap[1] = heap[size];     // move last to first
-        heap[size] = null;        // permit GC of objects
-        size--;
-        downHeap();               // adjust heap
-        return result;
-      } else {
-        return null;
-      }
-    }
-
-    /**
-     * Should be called when the Object at top changes values. Still log(n) worst
-     * case, but it's at least twice as fast to
-     *
-     * <pre class="prettyprint">
-     * pq.top().change();
-     * pq.updateTop();
-     * </pre>
-     *
-     * instead of
-     *
-     * <pre class="prettyprint">
-     * o = pq.pop();
-     * o.change();
-     * pq.push(o);
-     * </pre>
-     *
-     * @return the new 'top' element.
-     */
-    public final T updateTop() {
-      downHeap();
-      return heap[1];
-    }
-
-    /** Returns the number of elements currently stored in the PriorityQueue. */
-    public final int size() {
-      return size;
-    }
-
-    /** Removes all entries from the PriorityQueue. */
-    public final void clear() {
-      for (int i = 0; i <= size; i++) {
-        heap[i] = null;
-      }
-      size = 0;
-    }
-
-    private final void upHeap() {
-      int i = size;
-      T node = heap[i];          // save bottom node
-      int j = i >>> 1;
-      while (j > 0 && lessThan(node, heap[j])) {
-        heap[i] = heap[j];       // shift parents down
-        i = j;
-        j = j >>> 1;
-      }
-      heap[i] = node;            // install saved node
-    }
-
-    private final void downHeap() {
-      int i = 1;
-      T node = heap[i];          // save top node
-      int j = i << 1;            // find smaller child
-      int k = j + 1;
-      if (k <= size && lessThan(heap[k], heap[j])) {
-        j = k;
-      }
-      while (j <= size && lessThan(heap[j], node)) {
-        heap[i] = heap[j];       // shift up child
-        i = j;
-        j = i << 1;
-        k = j + 1;
-        if (k <= size && lessThan(heap[k], heap[j])) {
-          j = k;
-        }
-      }
-      heap[i] = node;            // install saved node
-    }
-
-    /** This method returns the internal heap array as Object[].
-     * @lucene.internal
-     */
-    public final Object[] getHeapArray() {
-      return (Object[]) heap;
-    }
-  }
-
-  public static class IgnoreException extends IOException {
-    public void printStackTrace(PrintWriter pw) {
-      pw.print("Early Client Disconnect");
-    }
-
-    public String getMessage() {
-      return "Early Client Disconnect";
-    }
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9d9c3a0c/solr/core/src/java/org/apache/solr/handler/export/BoolFieldWriter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/export/BoolFieldWriter.java b/solr/core/src/java/org/apache/solr/handler/export/BoolFieldWriter.java
new file mode 100644
index 0000000..e67f5ed
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/export/BoolFieldWriter.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.export;
+
+import java.io.IOException;
+
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.CharsRefBuilder;
+import org.apache.solr.common.MapWriter;
+import org.apache.solr.schema.FieldType;
+
+class BoolFieldWriter extends FieldWriter {
+  private String field;
+  private FieldType fieldType;
+  private CharsRefBuilder cref = new CharsRefBuilder();
+
+  public BoolFieldWriter(String field, FieldType fieldType) {
+    this.field = field;
+    this.fieldType = fieldType;
+  }
+
+  public boolean write(int docId, LeafReader reader, MapWriter.EntryWriter ew, int fieldIndex) throws IOException {
+    SortedDocValues vals = DocValues.getSorted(reader, this.field);
+    if (vals.advance(docId) != docId) {
+      return false;
+    }
+    int ord = vals.ordValue();
+
+    BytesRef ref = vals.lookupOrd(ord);
+    fieldType.indexedToReadable(ref, cref);
+    ew.put(this.field, "true".equals(cref.toString()));
+    return true;
+  }
+}

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9d9c3a0c/solr/core/src/java/org/apache/solr/handler/export/DateFieldWriter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/export/DateFieldWriter.java b/solr/core/src/java/org/apache/solr/handler/export/DateFieldWriter.java
new file mode 100644
index 0000000..3584317
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/export/DateFieldWriter.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.export;
+
+import java.io.IOException;
+import java.util.Date;
+
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.NumericDocValues;
+import org.apache.solr.common.MapWriter;
+
+class DateFieldWriter extends FieldWriter {
+  private String field;
+
+  public DateFieldWriter(String field) {
+    this.field = field;
+  }
+
+  public boolean write(int docId, LeafReader reader, MapWriter.EntryWriter ew, int fieldIndex) throws IOException {
+    NumericDocValues vals = DocValues.getNumeric(reader, this.field);
+    long val;
+    if (vals.advance(docId) == docId) {
+      val = vals.longValue();
+    } else {
+      return false;
+    }
+    ew.put(this.field, new Date(val));
+    return true;
+  }
+}

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9d9c3a0c/solr/core/src/java/org/apache/solr/handler/export/DoubeValue.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/export/DoubeValue.java b/solr/core/src/java/org/apache/solr/handler/export/DoubeValue.java
new file mode 100644
index 0000000..a37e13d
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/export/DoubeValue.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.export;
+
+import java.io.IOException;
+
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.NumericDocValues;
+
+class DoubleValue implements SortValue {
+
+  protected NumericDocValues vals;
+  protected String field;
+  protected double currentValue;
+  protected DoubleComp comp;
+  private int lastDocID;
+  private LeafReader reader;
+
+  public DoubleValue(String field, DoubleComp comp) {
+    this.field = field;
+    this.comp = comp;
+    this.currentValue = comp.resetValue();
+  }
+
+  public DoubleValue copy() {
+    return new DoubleValue(field, comp);
+  }
+
+  public void setNextReader(LeafReaderContext context) throws IOException {
+    this.reader = context.reader();
+    this.vals = DocValues.getNumeric(this.reader, this.field);
+    lastDocID = 0;
+  }
+
+  public void setCurrentValue(int docId) throws IOException {
+    if (docId < lastDocID) {
+      throw new AssertionError("docs were sent out-of-order: lastDocID=" + lastDocID + " vs doc=" + docId);
+    }
+    lastDocID = docId;
+    int curDocID = vals.docID();
+    if (docId > curDocID) {
+      curDocID = vals.advance(docId);
+    }
+    if (docId == curDocID) {
+      currentValue = Double.longBitsToDouble(vals.longValue());
+    } else {
+      currentValue = 0f;
+    }
+  }
+
+  public void setCurrentValue(SortValue sv) {
+    DoubleValue dv = (DoubleValue)sv;
+    this.currentValue = dv.currentValue;
+  }
+
+  public void reset() {
+    this.currentValue = comp.resetValue();
+  }
+
+  public int compareTo(SortValue o) {
+    DoubleValue dv = (DoubleValue)o;
+    return comp.compare(currentValue, dv.currentValue);
+  }
+}

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9d9c3a0c/solr/core/src/java/org/apache/solr/handler/export/DoubleCmp.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/export/DoubleCmp.java b/solr/core/src/java/org/apache/solr/handler/export/DoubleCmp.java
new file mode 100644
index 0000000..50341fd
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/export/DoubleCmp.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.export;
+
+interface DoubleComp {
+  int compare(double a, double b);
+  double resetValue();
+}
+
+class DoubleAsc implements DoubleComp {
+  public double resetValue() {
+    return Double.MAX_VALUE;
+  }
+
+  public int compare(double a, double b) {
+    return Double.compare(b, a);
+  }
+}
+
+class DoubleDesc implements DoubleComp {
+  public double resetValue() {
+    return -Double.MAX_VALUE;
+  }
+
+  public int compare(double a, double b) {
+    return Double.compare(a, b);
+  }
+}

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9d9c3a0c/solr/core/src/java/org/apache/solr/handler/export/DoubleFieldWriter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/export/DoubleFieldWriter.java b/solr/core/src/java/org/apache/solr/handler/export/DoubleFieldWriter.java
new file mode 100644
index 0000000..1b953c5
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/export/DoubleFieldWriter.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.export;
+
+import java.io.IOException;
+
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.NumericDocValues;
+import org.apache.solr.common.MapWriter;
+
+class DoubleFieldWriter extends FieldWriter {
+  private String field;
+
+  public DoubleFieldWriter(String field) {
+    this.field = field;
+  }
+
+  public boolean write(int docId, LeafReader reader, MapWriter.EntryWriter ew, int fieldIndex) throws IOException {
+    NumericDocValues vals = DocValues.getNumeric(reader, this.field);
+    long val;
+    if (vals.advance(docId) == docId) {
+      val = vals.longValue();
+    } else {
+      return false;
+    }
+    ew.put(this.field, Double.longBitsToDouble(val));
+    return true;
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9d9c3a0c/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java b/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java
new file mode 100644
index 0000000..78d0795
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java
@@ -0,0 +1,448 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.export;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.lang.invoke.MethodHandles;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.util.BitSetIterator;
+import org.apache.lucene.util.FixedBitSet;
+import org.apache.solr.client.solrj.impl.BinaryResponseParser;
+import org.apache.solr.common.IteratorWriter;
+import org.apache.solr.common.MapWriter;
+import org.apache.solr.common.MapWriter.EntryWriter;
+import org.apache.solr.common.PushWriter;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.JavaBinCodec;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.request.SolrRequestInfo;
+import org.apache.solr.response.BinaryResponseWriter;
+import org.apache.solr.response.JSONResponseWriter;
+import org.apache.solr.response.QueryResponseWriter;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.schema.BoolField;
+import org.apache.solr.schema.DateValueFieldType;
+import org.apache.solr.schema.DoubleValueFieldType;
+import org.apache.solr.schema.FieldType;
+import org.apache.solr.schema.FloatValueFieldType;
+import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.schema.IntValueFieldType;
+import org.apache.solr.schema.LongValueFieldType;
+import org.apache.solr.schema.SchemaField;
+import org.apache.solr.schema.StrField;
+import org.apache.solr.search.SolrIndexSearcher;
+import org.apache.solr.search.SortSpec;
+import org.apache.solr.search.SyntaxError;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static java.util.Collections.singletonList;
+import static java.util.Collections.singletonMap;
+import static org.apache.solr.common.util.Utils.makeMap;
+
+public class ExportWriter implements SolrCore.RawWriter, Closeable {
+  private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+  private OutputStreamWriter respWriter;
+  final SolrQueryRequest req;
+  final SolrQueryResponse res;
+  FieldWriter[] fieldWriters;
+  int totalHits = 0;
+  FixedBitSet[] sets = null;
+  PushWriter writer;
+  private String wt;
+
+
+  public ExportWriter(SolrQueryRequest req, SolrQueryResponse res, String wt) {
+    this.req = req;
+    this.res = res;
+    this.wt = wt;
+
+  }
+
+  @Override
+  public String getContentType() {
+    if ("javabin".equals(wt)) {
+      return BinaryResponseParser.BINARY_CONTENT_TYPE;
+    } else return "json";
+  }
+
+  @Override
+  public void close() throws IOException {
+    if (writer != null) writer.close();
+    if (respWriter != null) {
+      respWriter.flush();
+      respWriter.close();
+    }
+
+  }
+
+  protected void writeException(Exception e, PushWriter w, boolean log) throws IOException {
+    w.writeMap(mw -> {
+      mw.put("responseHeader", singletonMap("status", 400))
+          .put("response", makeMap(
+              "numFound", 0,
+              "docs", singletonList(singletonMap("EXCEPTION", e.getMessage()))));
+    });
+    if (log) {
+      SolrException.log(logger, e);
+    }
+  }
+
+  public void write(OutputStream os) throws IOException {
+    QueryResponseWriter rw = req.getCore().getResponseWriters().get(wt);
+    if (rw instanceof BinaryResponseWriter) {
+      //todo add support for other writers after testing
+      writer = new JavaBinCodec(os, null);
+    } else {
+      respWriter = new OutputStreamWriter(os, StandardCharsets.UTF_8);
+      writer = JSONResponseWriter.getPushWriter(respWriter, req, res);
+    }
+    Exception exception = res.getException();
+    if (exception != null) {
+      if (!(exception instanceof IgnoreException)) {
+        writeException(exception, writer, false);
+      }
+      return;
+    }
+
+    SolrRequestInfo info = SolrRequestInfo.getRequestInfo();
+    SortSpec sortSpec = info.getResponseBuilder().getSortSpec();
+
+    if (sortSpec == null) {
+      writeException((new IOException(new SyntaxError("No sort criteria was provided."))), writer, true);
+      return;
+    }
+
+    SolrIndexSearcher searcher = req.getSearcher();
+    Sort sort = searcher.weightSort(sortSpec.getSort());
+
+    if (sort == null) {
+      writeException((new IOException(new SyntaxError("No sort criteria was provided."))), writer, true);
+      return;
+    }
+
+    if (sort != null && sort.needsScores()) {
+      writeException((new IOException(new SyntaxError("Scoring is not currently supported with xsort."))), writer, true);
+      return;
+    }
+
+    // There is a bailout in SolrIndexSearcher.getDocListNC when there are _no_ docs in the index at all.
+    // if (lastDocRequested <= 0) {
+    // That causes the totalHits and export entries in the context to _not_ get set.
+    // The only time that really matters is when we search against an _empty_ set. That's too obscure
+    // a condition to handle as part of this patch, if someone wants to pursue it it can be reproduced with:
+    // ant test  -Dtestcase=StreamingTest -Dtests.method=testAllValidExportTypes -Dtests.seed=10F13879D0D1D6AD -Dtests.slow=true -Dtests.locale=es-PA -Dtests.timezone=America/Bahia_Banderas -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
+    // You'll have to uncomment the if below to hit the null pointer exception.
+    // This is such an unusual case (i.e. an empty index) that catching this concdition here is probably OK.
+    // This came to light in the very artifical case of indexing a single doc to Cloud.
+    if (req.getContext().get("totalHits") != null) {
+      totalHits = ((Integer) req.getContext().get("totalHits")).intValue();
+      sets = (FixedBitSet[]) req.getContext().get("export");
+      if (sets == null) {
+        writeException((new IOException(new SyntaxError("xport RankQuery is required for xsort: rq={!xport}"))), writer, true);
+        return;
+      }
+    }
+    SolrParams params = req.getParams();
+    String fl = params.get("fl");
+
+    String[] fields = null;
+
+    if (fl == null) {
+      writeException((new IOException(new SyntaxError("export field list (fl) must be specified."))), writer, true);
+      return;
+    } else {
+      fields = fl.split(",");
+
+      for (int i = 0; i < fields.length; i++) {
+
+        fields[i] = fields[i].trim();
+
+        if (fields[i].equals("score")) {
+          writeException((new IOException(new SyntaxError("Scoring is not currently supported with xsort."))), writer, true);
+          return;
+        }
+      }
+    }
+
+    try {
+      fieldWriters = getFieldWriters(fields, req.getSearcher());
+    } catch (Exception e) {
+      writeException(e, writer, true);
+      return;
+    }
+
+    writer.writeMap(m -> {
+      m.put("responseHeader", singletonMap("status", 0));
+      m.put("response", (MapWriter) mw -> {
+        mw.put("numFound", totalHits);
+        mw.put("docs", (IteratorWriter) iw -> writeDocs(req, iw, sort));
+      });
+    });
+
+  }
+
+  protected void writeDocs(SolrQueryRequest req, IteratorWriter.ItemWriter writer, Sort sort) throws IOException {
+    //Write the data.
+    List<LeafReaderContext> leaves = req.getSearcher().getTopReaderContext().leaves();
+    SortDoc sortDoc = getSortDoc(req.getSearcher(), sort.getSort());
+    int count = 0;
+    int queueSize = 30000;
+    if (totalHits < 30000) {
+      queueSize = totalHits;
+    }
+    SortQueue queue = new SortQueue(queueSize, sortDoc);
+    SortDoc[] outDocs = new SortDoc[queueSize];
+
+    while (count < totalHits) {
+      //long begin = System.nanoTime();
+      queue.reset();
+      SortDoc top = queue.top();
+      for (int i = 0; i < leaves.size(); i++) {
+        sortDoc.setNextReader(leaves.get(i));
+        DocIdSetIterator it = new BitSetIterator(sets[i], 0); // cost is not useful here
+        int docId;
+        while ((docId = it.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
+          sortDoc.setValues(docId);
+          if (top.lessThan(sortDoc)) {
+            top.setValues(sortDoc);
+            top = queue.updateTop();
+          }
+        }
+      }
+
+      int outDocsIndex = -1;
+
+      for (int i = 0; i < queueSize; i++) {
+        SortDoc s = queue.pop();
+        if (s.docId > -1) {
+          outDocs[++outDocsIndex] = s;
+        }
+      }
+
+      //long end = System.nanoTime();
+
+      count += (outDocsIndex + 1);
+
+      try {
+        for (int i = outDocsIndex; i >= 0; --i) {
+          SortDoc s = outDocs[i];
+          writer.add((MapWriter) ew -> {
+            writeDoc(s, leaves, ew);
+            s.reset();
+          });
+        }
+      } catch (Throwable e) {
+        Throwable ex = e;
+        while (ex != null) {
+          String m = ex.getMessage();
+          if (m != null && m.contains("Broken pipe")) {
+            throw new IgnoreException();
+          }
+          ex = ex.getCause();
+        }
+
+        if (e instanceof IOException) {
+          throw ((IOException) e);
+        } else {
+          throw new IOException(e);
+        }
+      }
+    }
+  }
+
+  protected void writeDoc(SortDoc sortDoc,
+                          List<LeafReaderContext> leaves,
+                          EntryWriter ew) throws IOException {
+
+    int ord = sortDoc.ord;
+    FixedBitSet set = sets[ord];
+    set.clear(sortDoc.docId);
+    LeafReaderContext context = leaves.get(ord);
+    int fieldIndex = 0;
+    for (FieldWriter fieldWriter : fieldWriters) {
+      if (fieldWriter.write(sortDoc.docId, context.reader(), ew, fieldIndex)) {
+        ++fieldIndex;
+      }
+    }
+  }
+
+  protected FieldWriter[] getFieldWriters(String[] fields, SolrIndexSearcher searcher) throws IOException {
+    IndexSchema schema = searcher.getSchema();
+    FieldWriter[] writers = new FieldWriter[fields.length];
+    for (int i = 0; i < fields.length; i++) {
+      String field = fields[i];
+      SchemaField schemaField = null;
+
+      try {
+        schemaField = schema.getField(field);
+      } catch (Exception e) {
+        throw new IOException(e);
+      }
+
+      if (!schemaField.hasDocValues()) {
+        throw new IOException(field + " must have DocValues to use this feature.");
+      }
+
+      boolean multiValued = schemaField.multiValued();
+      FieldType fieldType = schemaField.getType();
+      if (fieldType instanceof IntValueFieldType) {
+        if (multiValued) {
+          writers[i] = new MultiFieldWriter(field, fieldType, schemaField, true);
+        } else {
+          writers[i] = new IntFieldWriter(field);
+        }
+      } else if (fieldType instanceof LongValueFieldType) {
+        if (multiValued) {
+          writers[i] = new MultiFieldWriter(field, fieldType, schemaField, true);
+        } else {
+          writers[i] = new LongFieldWriter(field);
+        }
+      } else if (fieldType instanceof FloatValueFieldType) {
+        if (multiValued) {
+          writers[i] = new MultiFieldWriter(field, fieldType, schemaField, true);
+        } else {
+          writers[i] = new FloatFieldWriter(field);
+        }
+      } else if (fieldType instanceof DoubleValueFieldType) {
+        if (multiValued) {
+          writers[i] = new MultiFieldWriter(field, fieldType, schemaField, true);
+        } else {
+          writers[i] = new DoubleFieldWriter(field);
+        }
+      } else if (fieldType instanceof StrField) {
+        if (multiValued) {
+          writers[i] = new MultiFieldWriter(field, fieldType, schemaField, false);
+        } else {
+          writers[i] = new StringFieldWriter(field, fieldType);
+        }
+      } else if (fieldType instanceof DateValueFieldType) {
+        if (multiValued) {
+          writers[i] = new MultiFieldWriter(field, fieldType, schemaField, false);
+        } else {
+          writers[i] = new DateFieldWriter(field);
+        }
+      } else if (fieldType instanceof BoolField) {
+        if (multiValued) {
+          writers[i] = new MultiFieldWriter(field, fieldType, schemaField, true);
+        } else {
+          writers[i] = new BoolFieldWriter(field, fieldType);
+        }
+      } else {
+        throw new IOException("Export fields must either be one of the following types: int,float,long,double,string,date,boolean");
+      }
+    }
+    return writers;
+  }
+
+  private SortDoc getSortDoc(SolrIndexSearcher searcher, SortField[] sortFields) throws IOException {
+    SortValue[] sortValues = new SortValue[sortFields.length];
+    IndexSchema schema = searcher.getSchema();
+    for (int i = 0; i < sortFields.length; ++i) {
+      SortField sf = sortFields[i];
+      String field = sf.getField();
+      boolean reverse = sf.getReverse();
+      SchemaField schemaField = schema.getField(field);
+      FieldType ft = schemaField.getType();
+
+      if (!schemaField.hasDocValues()) {
+        throw new IOException(field + " must have DocValues to use this feature.");
+      }
+
+      if (ft instanceof IntValueFieldType) {
+        if (reverse) {
+          sortValues[i] = new IntValue(field, new IntDesc());
+        } else {
+          sortValues[i] = new IntValue(field, new IntAsc());
+        }
+      } else if (ft instanceof FloatValueFieldType) {
+        if (reverse) {
+          sortValues[i] = new FloatValue(field, new FloatDesc());
+        } else {
+          sortValues[i] = new FloatValue(field, new FloatAsc());
+        }
+      } else if (ft instanceof DoubleValueFieldType) {
+        if (reverse) {
+          sortValues[i] = new DoubleValue(field, new DoubleDesc());
+        } else {
+          sortValues[i] = new DoubleValue(field, new DoubleAsc());
+        }
+      } else if (ft instanceof LongValueFieldType) {
+        if (reverse) {
+          sortValues[i] = new LongValue(field, new LongDesc());
+        } else {
+          sortValues[i] = new LongValue(field, new LongAsc());
+        }
+      } else if (ft instanceof StrField) {
+        LeafReader reader = searcher.getSlowAtomicReader();
+        SortedDocValues vals = reader.getSortedDocValues(field);
+        if (reverse) {
+          sortValues[i] = new StringValue(vals, field, new IntDesc());
+        } else {
+          sortValues[i] = new StringValue(vals, field, new IntAsc());
+        }
+      } else if (ft instanceof DateValueFieldType) {
+        if (reverse) {
+          sortValues[i] = new LongValue(field, new LongDesc());
+        } else {
+          sortValues[i] = new LongValue(field, new LongAsc());
+        }
+      } else if (ft instanceof BoolField) {
+        // This is a bit of a hack, but since the boolean field stores ByteRefs, just like Strings
+        // _and_ since "F" happens to sort before "T" (thus false sorts "less" than true)
+        // we can just use the existing StringValue here.
+        LeafReader reader = searcher.getSlowAtomicReader();
+        SortedDocValues vals = reader.getSortedDocValues(field);
+        if (reverse) {
+          sortValues[i] = new StringValue(vals, field, new IntDesc());
+        } else {
+          sortValues[i] = new StringValue(vals, field, new IntAsc());
+        }
+      } else {
+        throw new IOException("Sort fields must be one of the following types: int,float,long,double,string,date,boolean");
+      }
+    }
+
+    return new SortDoc(sortValues);
+  }
+
+  public static class IgnoreException extends IOException {
+    public void printStackTrace(PrintWriter pw) {
+      pw.print("Early Client Disconnect");
+    }
+
+    public String getMessage() {
+      return "Early Client Disconnect";
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9d9c3a0c/solr/core/src/java/org/apache/solr/handler/export/FieldWriter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/export/FieldWriter.java b/solr/core/src/java/org/apache/solr/handler/export/FieldWriter.java
new file mode 100644
index 0000000..bd69fc0
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/export/FieldWriter.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.export;
+
+import java.io.IOException;
+
+import org.apache.lucene.index.LeafReader;
+import org.apache.solr.common.MapWriter;
+
+abstract class FieldWriter {
+  public abstract boolean write(int docId, LeafReader reader, MapWriter.EntryWriter out, int fieldIndex) throws IOException;
+}
\ No newline at end of file


Mime
View raw message