lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From is...@apache.org
Subject [lucene-solr] 01/01: SOLR-7414: Adding explicitly required fields support to CSV and XLSX response writers
Date Mon, 18 Feb 2019 10:21:09 GMT
This is an automated email from the ASF dual-hosted git repository.

ishan pushed a commit to branch SOLR-7414
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git

commit 6911c86c0217a50be266fad25de188d132f07127
Author: Ishan Chattopadhyaya <ishan@apache.org>
AuthorDate: Mon Feb 18 15:50:44 2019 +0530

    SOLR-7414: Adding explicitly required fields support to CSV and XLSX response writers
---
 .../handler/extraction/XLSXResponseWriter.java     | 27 +++++++++++++++------
 .../apache/solr/response/CSVResponseWriter.java    | 22 ++++++++++++-----
 .../java/org/apache/solr/search/ReturnFields.java  |  7 ++++++
 .../org/apache/solr/search/SolrReturnFields.java   | 28 ++++++++++++++--------
 .../solr/response/TestCSVResponseWriter.java       | 22 ++++++++++++++++-
 5 files changed, 82 insertions(+), 24 deletions(-)

diff --git a/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/XLSXResponseWriter.java
b/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/XLSXResponseWriter.java
index baa1ddb..96b34e2 100644
--- a/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/XLSXResponseWriter.java
+++ b/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/XLSXResponseWriter.java
@@ -30,6 +30,7 @@ import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Sets;
@@ -178,13 +179,18 @@ class XLSXWriter extends TextResponseWriter {
     this.rsp = rsp;
   }
 
+  // TODO: a lot of the code here is copied from CSVResponseWriter. We should look at
+  // refactoring into a common place.
+  // nocommit: Added support for explicitly requested fields (SOLR-7414) similar to how it
+  // was done at CSVResponseWriter. However, this hasn't been tested here.
   public void writeResponse(OutputStream out, LinkedHashMap<String, String> colNamesMap,
                             LinkedHashMap<String, Integer> colWidthsMap) throws IOException
{
     SolrParams params = req.getParams();
 
     Collection<String> fields = returnFields.getRequestedFieldNames();
+    Set<String> explicitReqFields = returnFields.getExplicitlyRequestedFieldNames();
     Object responseObj = rsp.getValues().get("response");
-    boolean returnOnlyStored = false;
+    boolean returnStoredOrDocValStored = false;
     if (fields==null||returnFields.hasPatternMatching()) {
       if (responseObj instanceof SolrDocumentList) {
         // get the list of fields from the SolrDocumentList
@@ -203,12 +209,17 @@ class XLSXWriter extends TextResponseWriter {
           Iterables.addAll(fields, all);
         }
       }
+      
+      if (explicitReqFields != null) {
+        // add explicit requested fields
+        Iterables.addAll(fields, explicitReqFields);
+      }
       if (returnFields.wantsScore()) {
         fields.add("score");
       } else {
         fields.remove("score");
       }
-      returnOnlyStored = true;
+      returnStoredOrDocValStored = true;
     }
 
     for (String field : fields) {
@@ -223,16 +234,18 @@ class XLSXWriter extends TextResponseWriter {
       }
 
       SchemaField sf = schema.getFieldOrNull(field);
+      // Return stored fields or useDocValuesAsStored=true fields,
+      // unless an explicit field list is specified
+      if (returnStoredOrDocValStored && !(explicitReqFields != null && explicitReqFields.contains(field))
&&
+          sf!= null && !sf.stored() && !(sf.hasDocValues() && sf.useDocValuesAsStored()))
{
+        continue;
+      }
+
       if (sf == null) {
         FieldType ft = new StrField();
         sf = new SchemaField(field, ft);
       }
 
-      // Return only stored fields, unless an explicit field list is specified
-      if (returnOnlyStored && sf != null && !sf.stored()) {
-        continue;
-      }
-
       XLField xlField = new XLField();
       xlField.name = field;
       xlField.sf = sf;
diff --git a/solr/core/src/java/org/apache/solr/response/CSVResponseWriter.java b/solr/core/src/java/org/apache/solr/response/CSVResponseWriter.java
index 5ebec77..aad4069 100644
--- a/solr/core/src/java/org/apache/solr/response/CSVResponseWriter.java
+++ b/solr/core/src/java/org/apache/solr/response/CSVResponseWriter.java
@@ -28,6 +28,7 @@ import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Sets;
@@ -246,6 +247,7 @@ class CSVWriter extends TextResponseWriter {
     }
 
     Collection<String> fields = returnFields.getRequestedFieldNames();
+    Set<String> explicitReqFields = returnFields.getExplicitlyRequestedFieldNames();
     Object responseObj = rsp.getResponse();
     boolean returnStoredOrDocValStored = false;
     if (fields==null||returnFields.hasPatternMatching()) {
@@ -266,6 +268,12 @@ class CSVWriter extends TextResponseWriter {
           Iterables.addAll(fields, all);
         }
       }
+
+      if (explicitReqFields != null) {
+        // add explicit requested fields
+        Iterables.addAll(fields, explicitReqFields);
+      }
+
       if (returnFields.wantsScore()) {
         fields.add("score");
       } else {
@@ -288,17 +296,19 @@ class CSVWriter extends TextResponseWriter {
       }
 
       SchemaField sf = schema.getFieldOrNull(field);
-      if (sf == null) {
-        FieldType ft = new StrField();
-        sf = new SchemaField(field, ft);
-      }
-      
+
       // Return stored fields or useDocValuesAsStored=true fields,
       // unless an explicit field list is specified
-      if (returnStoredOrDocValStored && !sf.stored() && !(sf.hasDocValues()
&& sf.useDocValuesAsStored())) {
+      if (returnStoredOrDocValStored && !(explicitReqFields != null && explicitReqFields.contains(field))
&&
+          sf!= null && !sf.stored() && !(sf.hasDocValues() && sf.useDocValuesAsStored()))
{
         continue;
       }
 
+      if (sf == null) {
+        FieldType ft = new StrField();
+        sf = new SchemaField(field, ft);
+      }
+
       // check for per-field overrides
       sep = params.get("f." + field + '.' + CSV_SEPARATOR);
       encapsulator = params.get("f." + field + '.' + CSV_ENCAPSULATOR);
diff --git a/solr/core/src/java/org/apache/solr/search/ReturnFields.java b/solr/core/src/java/org/apache/solr/search/ReturnFields.java
index c2d5973..0c35ea5 100644
--- a/solr/core/src/java/org/apache/solr/search/ReturnFields.java
+++ b/solr/core/src/java/org/apache/solr/search/ReturnFields.java
@@ -60,6 +60,13 @@ public abstract class ReturnFields {
   public abstract Set<String> getRequestedFieldNames();
 
   /**
+   * The explicitly requested field names (includes pseudo fields)
+   * <p>
+   * @return Set of explicitly requested field names or <code>null</code> (no
explicit)
+   */
+  public abstract Set<String> getExplicitlyRequestedFieldNames();
+
+  /**
    * Get the fields which have been renamed
    * @return a mapping of renamed fields
    */
diff --git a/solr/core/src/java/org/apache/solr/search/SolrReturnFields.java b/solr/core/src/java/org/apache/solr/search/SolrReturnFields.java
index d9e681b..9a04c25 100644
--- a/solr/core/src/java/org/apache/solr/search/SolrReturnFields.java
+++ b/solr/core/src/java/org/apache/solr/search/SolrReturnFields.java
@@ -16,6 +16,15 @@
  */
 package org.apache.solr.search;
 
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Supplier;
+
 import org.apache.commons.io.FilenameUtils;
 import org.apache.lucene.queries.function.FunctionQuery;
 import org.apache.lucene.queries.function.ValueSource;
@@ -35,15 +44,6 @@ import org.apache.solr.response.transform.TransformerFactory;
 import org.apache.solr.response.transform.ValueSourceAugmenter;
 import org.apache.solr.search.SolrDocumentFetcher.RetrieveFieldsOptimizer;
 
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.function.Supplier;
-
 /**
  * The default implementation of return fields parsing for Solr.
  */
@@ -448,7 +448,15 @@ public class SolrReturnFields extends ReturnFields {
 
   @Override
   public Set<String> getRequestedFieldNames() {
-    if(_wantsAllFields || reqFieldNames==null || reqFieldNames.isEmpty()) {
+    if(_wantsAllFields || reqFieldNames == null || reqFieldNames.isEmpty()) {
+      return null;
+    }
+    return reqFieldNames;
+  }
+
+  @Override
+  public Set<String> getExplicitlyRequestedFieldNames() {
+    if (reqFieldNames == null || reqFieldNames.isEmpty()) {
       return null;
     }
     return reqFieldNames;
diff --git a/solr/core/src/test/org/apache/solr/response/TestCSVResponseWriter.java b/solr/core/src/test/org/apache/solr/response/TestCSVResponseWriter.java
index 979279c..204496a 100644
--- a/solr/core/src/test/org/apache/solr/response/TestCSVResponseWriter.java
+++ b/solr/core/src/test/org/apache/solr/response/TestCSVResponseWriter.java
@@ -42,7 +42,7 @@ public class TestCSVResponseWriter extends SolrTestCaseJ4 {
   public static void createIndex() {
     assertU(adoc("id","1", "foo_i","-1", "foo_s","hi", "foo_l","12345678987654321", "foo_b","false",
"foo_f","1.414","foo_d","-1.0E300","foo_dt","2000-01-02T03:04:05Z"));
     assertU(adoc("id","2", "v_ss","hi",  "v_ss","there", "v2_ss","nice", "v2_ss","output",
"shouldbeunstored","foo"));
-    assertU(adoc("id","3", "shouldbeunstored","foo"));
+    assertU(adoc("id","3", "shouldbeunstored","foo", "foo_l", "1"));
     assertU(adoc("id","4", "amount_c", "1.50,EUR"));
     assertU(adoc("id","5", "store", "12.434,-134.1"));
     assertU(adoc("id","6", "pubyear_ii", "123", "store_iis", "12", "price_ff", "1.3"));
@@ -246,6 +246,13 @@ public class TestCSVResponseWriter extends SolrTestCaseJ4 {
     assertEquals("exists(shouldbeunstored),XXX", singleFuncLines[0] );
     assertEquals("false,1", singleFuncLines[1] );
     assertEquals("true,3", singleFuncLines[3] );
+
+    // pseudo-fields with * in fl
+    txt = h.query(req("q","id:4", "wt","csv", "csv.header","true", "fl","*,YYY:[docid],FOO:amount_c"));
+    lines = txt.split("\n");
+    assertEquals(2, lines.length);
+    assertEquals(sortHeader("foo_i,foo_l,FOO,foo_s,store,store_iis," +
+        "v2_ss,pubyear_ii,foo_dt,foo_b,YYY,foo_d,id,amount_c,foo_f,v_ss"), sortHeader(lines[0]));
   }
 
   @Test
@@ -283,6 +290,19 @@ public class TestCSVResponseWriter extends SolrTestCaseJ4 {
     assertEquals(2, singleFuncLines.length);
     assertEquals("price_ff", singleFuncLines[0]);
     assertEquals("1.3", singleFuncLines[1]);
+
+    // explicit price_ff with fl=*
+    singleFuncText = h.query(req("q","id:6", "wt","csv", "csv.header","true", "fl", "*,price_ff"));
+    sortedHeader = sortHeader("amount_c,store,v_ss,foo_b,v2_ss,foo_f,foo_i,foo_d,foo_s,foo_dt,id,foo_l,"
+
+        "pubyear_ii,store_iis,price_ff");
+    singleFuncLines = singleFuncText.split("\n");
+    assertEquals(2, singleFuncLines.length);
+    assertEquals(sortedHeader, sortHeader(singleFuncLines[0]));
+    actualVal = Arrays.stream(singleFuncLines[1].trim().split(","))
+        .filter(val -> !val.trim().isEmpty() && !val.trim().equals("\"\""))
+        .collect(Collectors.toList());
+    assertEquals(4, actualVal.size());
+    assertTrue(actualVal.containsAll(Arrays.asList("6", "123", "12", "1.3")));
   }
     
 


Mime
View raw message