lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From yo...@apache.org
Subject lucene-solr:master: SOLR-8865: Real-time get sometimes fails to retrieve stored fields from docValues
Date Fri, 25 Mar 2016 14:18:44 GMT
Repository: lucene-solr
Updated Branches:
  refs/heads/master c498fcdc4 -> 6f60ac21f


SOLR-8865: Real-time get sometimes fails to retrieve stored fields from docValues


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/6f60ac21
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/6f60ac21
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/6f60ac21

Branch: refs/heads/master
Commit: 6f60ac21f303364eece9c9f2893278f7da31aef8
Parents: c498fcd
Author: yonik <yonik@apache.org>
Authored: Fri Mar 25 10:18:31 2016 -0400
Committer: yonik <yonik@apache.org>
Committed: Fri Mar 25 10:18:31 2016 -0400

----------------------------------------------------------------------
 solr/CHANGES.txt                                |  3 ++
 .../handler/component/RealTimeGetComponent.java | 13 ++++-
 .../solr/collection1/conf/schema15.xml          |  1 +
 .../solr/collection1/conf/schema_latest.xml     | 51 +++++++++++++-------
 .../org/apache/solr/search/TestRealTimeGet.java | 49 +++++++++++++++++--
 5 files changed, 93 insertions(+), 24 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6f60ac21/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index fee5019..52e7c31 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -361,6 +361,9 @@ Bug Fixes
 * SOLR-8891: Fix StrField.toObject and toExternal to work with docValue IndexableField
   instances. (yonik)
 
+* SOLR-8865: Real-time get sometimes fails to retrieve stored fields from docValues.
+  (Ishan Chattopadhyaya, yonik)
+
 Optimizations
 ----------------------
 * SOLR-7876: Speed up queries and operations that use many terms when timeAllowed has not
been

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6f60ac21/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java
b/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java
index 2f71b3e..8332c79 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java
@@ -25,6 +25,7 @@ import java.util.List;
 import java.util.Map;
 
 import org.apache.lucene.document.Document;
+import org.apache.lucene.index.DocValuesType;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.LeafReaderContext;
@@ -258,6 +259,7 @@ public class RealTimeGetComponent extends SearchComponent
        if (docid < 0) continue;
        Document luceneDocument = searcher.doc(docid, rsp.getReturnFields().getLuceneFieldNames());
        SolrDocument doc = toSolrDoc(luceneDocument,  core.getLatestSchema());
+       searcher.decorateDocValueFields(doc, docid, searcher.getNonStoredDVs(true));
        if( transformer != null ) {
          transformer.transform(doc, docid, 0);
        }
@@ -408,8 +410,15 @@ public class RealTimeGetComponent extends SearchComponent
     // copy the stored fields only
     Document out = new Document();
     for (IndexableField f : doc.getFields()) {
-      if (f.fieldType().stored() ) {
-        out.add((IndexableField) f);
+      if (f.fieldType().stored()) {
+        out.add(f);
+      } else if (f.fieldType().docValuesType() != DocValuesType.NONE) {
+        SchemaField schemaField = schema.getFieldOrNull(f.name());
+        if (schemaField != null && !schemaField.stored() && schemaField.useDocValuesAsStored())
{
+          out.add(f);
+        }
+      } else {
+        log.debug("Don't know how to handle field " + f);
       }
     }
 

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6f60ac21/solr/core/src/test-files/solr/collection1/conf/schema15.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/schema15.xml b/solr/core/src/test-files/solr/collection1/conf/schema15.xml
index ea46d85..55ec3f1 100644
--- a/solr/core/src/test-files/solr/collection1/conf/schema15.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/schema15.xml
@@ -530,6 +530,7 @@
    <dynamicField name="*_s"  type="string"  indexed="true"  stored="true" multiValued="true"/>
    <dynamicField name="*_ss"  type="string"  indexed="true"  stored="true" multiValued="true"/>
    <dynamicField name="*_l"  type="long"   indexed="true"  stored="true"/>
+   <dynamicField name="*_ll" type="long"   indexed="true"  stored="true"  multiValued="true"/>
    <dynamicField name="*_t"  type="text"    indexed="true"  stored="true"/>
    <dynamicField name="*_tt"  type="text"    indexed="true"  stored="true"/>
    <dynamicField name="*_ws"  type="nametext" indexed="true" stored="true"/>

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6f60ac21/solr/core/src/test-files/solr/collection1/conf/schema_latest.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/schema_latest.xml b/solr/core/src/test-files/solr/collection1/conf/schema_latest.xml
index ef82aa1..cae373c 100644
--- a/solr/core/src/test-files/solr/collection1/conf/schema_latest.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/schema_latest.xml
@@ -110,7 +110,7 @@
    <!-- If you remove this field, you must _also_ disable the update log in solrconfig.xml
       or Solr won't start. _version_ and update log are required for SolrCloud
    --> 
-   <field name="_version_" type="long" indexed="true" stored="true"/>
+   <field name="_version_" type="long" indexed="false" stored="false" docValues="true"
/>
    
    <!-- points to the root document of a block of nested documents. Required for nested
       document support, may be removed otherwise
@@ -199,27 +199,48 @@
        RESTRICTION: the glob-like pattern in the name attribute must have
        a "*" only at the start or the end.  -->
    
+   <!-- docvalues and stored are exclusive -->
    <dynamicField name="*_i"  type="int"    indexed="true"  stored="true"/>
    <dynamicField name="*_is" type="int"    indexed="true"  stored="true"  multiValued="true"/>
-   <dynamicField name="*_id"  type="int"    indexed="true"  stored="true"            
           docValues="true" />
-   <dynamicField name="*_ids" type="int"    indexed="true"  stored="true"  multiValued="true"
   docValues="true" />
+   <dynamicField name="*_id"  type="int"    indexed="true"  stored="false"           
            docValues="true" />
+   <dynamicField name="*_ids" type="int"    indexed="true"  stored="false"  multiValued="true"
   docValues="true" />
    <dynamicField name="*_s"  type="string"  indexed="true"  stored="true" />
    <dynamicField name="*_s1"  type="string"  indexed="true"  stored="true" />
    <dynamicField name="*_ss" type="string"  indexed="true"  stored="true" multiValued="true"/>
-   <dynamicField name="*_sd"  type="string"  indexed="true"  stored="true"           
           docValues="true" />
-   <dynamicField name="*_sds" type="string"  indexed="true"  stored="true" multiValued="true"
   docValues="true" />
+   <dynamicField name="*_sd"  type="string"  indexed="true"  stored="false"          
            docValues="true" />
+   <dynamicField name="*_sds" type="string"  indexed="true"  stored="false" multiValued="true"
   docValues="true" />
    <dynamicField name="*_l"  type="long"   indexed="true"  stored="true"/>
    <dynamicField name="*_ls" type="long"   indexed="true"  stored="true"  multiValued="true"/>
-   <dynamicField name="*_ld"  type="long"   indexed="true"  stored="true"            
           docValues="true" />
-   <dynamicField name="*_lds" type="long"   indexed="true"  stored="true"  multiValued="true"
   docValues="true" />
+   <dynamicField name="*_ld"  type="long"   indexed="true"  stored="false"           
            docValues="true" />
+   <dynamicField name="*_lds" type="long"   indexed="true"  stored="false"  multiValued="true"
   docValues="true" />
    <dynamicField name="*_f"  type="float"  indexed="true"  stored="true"/>
    <dynamicField name="*_fs" type="float"  indexed="true"  stored="true"  multiValued="true"/>
-   <dynamicField name="*_fd"  type="float"  indexed="true"  stored="true"            
           docValues="true" />
-   <dynamicField name="*_fds" type="float"  indexed="true"  stored="true"  multiValued="true"
   docValues="true" />
+   <dynamicField name="*_fd"  type="float"  indexed="true"  stored="false"           
            docValues="true" />
+   <dynamicField name="*_fds" type="float"  indexed="true"  stored="false"  multiValued="true"
   docValues="true" />
    <dynamicField name="*_d"  type="double" indexed="true"  stored="true"/>
    <dynamicField name="*_ds" type="double" indexed="true"  stored="true"  multiValued="true"/>
-   <dynamicField name="*_dd"  type="double" indexed="true"  stored="true"            
           docValues="true" />
-   <dynamicField name="*_dds" type="double" indexed="true"  stored="true"  multiValued="true"
   docValues="true" />
+   <dynamicField name="*_dd"  type="double" indexed="true"  stored="false"           
            docValues="true" />
+   <dynamicField name="*_dds" type="double" indexed="true"  stored="false"  multiValued="true"
   docValues="true" />
+   <dynamicField name="*_dt"   type="date"   indexed="true"  stored="true"/>
+   <dynamicField name="*_dts"  type="date"   indexed="true"  stored="true" multiValued="true"/>
+   <dynamicField name="*_dtd"  type="date"   indexed="true"  stored="false"          
            docValues="true" />
+   <dynamicField name="*_dtds" type="date"   indexed="true"  stored="false" multiValued="true"
   docValues="true" />
+ 
+   <!-- docvalues and stored (S suffix) -->
+   <dynamicField name="*_idS"  type="int"    indexed="true"  stored="true"           
            docValues="true" />
+   <dynamicField name="*_idsS" type="int"    indexed="true"  stored="true"  multiValued="true"
   docValues="true" />
+   <dynamicField name="*_sdS"  type="string"  indexed="true"  stored="true"          
            docValues="true" />
+   <dynamicField name="*_sdsS" type="string"  indexed="true"  stored="true" multiValued="true"
   docValues="true" />
+   <dynamicField name="*_ldS"  type="long"   indexed="true"  stored="true"           
            docValues="true" />
+   <dynamicField name="*_ldsS" type="long"   indexed="true"  stored="true"  multiValued="true"
   docValues="true" />
+   <dynamicField name="*_fdS"  type="float"  indexed="true"  stored="true"           
            docValues="true" />
+   <dynamicField name="*_fdsS" type="float"  indexed="true"  stored="true"  multiValued="true"
   docValues="true" />
+   <dynamicField name="*_ddS"  type="double" indexed="true"  stored="true"           
            docValues="true" />
+   <dynamicField name="*_ddsS" type="double" indexed="true"  stored="true"  multiValued="true"
   docValues="true" />
+   <dynamicField name="*_dtdS"  type="date"   indexed="true"  stored="true"          
            docValues="true" />
+   <dynamicField name="*_dtdsS" type="date"   indexed="true"  stored="true" multiValued="true"
   docValues="true" />
+
+
    <dynamicField name="*_b"  type="boolean" indexed="true" stored="true"/>
    <dynamicField name="*_bs" type="boolean" indexed="true" stored="true"  multiValued="true"/>
 
@@ -230,10 +251,6 @@
    <!-- Type used to index the lat and lon components for the "location" FieldType -->
    <dynamicField name="*_coordinate"  type="tdouble" indexed="true"  stored="false" />
 
-   <dynamicField name="*_dt"   type="date"   indexed="true"  stored="true"/>
-   <dynamicField name="*_dts"  type="date"   indexed="true"  stored="true" multiValued="true"/>
-   <dynamicField name="*_dtd"  type="date"   indexed="true"  stored="true"           
           docValues="true" />
-   <dynamicField name="*_dtds" type="date"   indexed="true"  stored="true" multiValued="true"
   docValues="true" />
    <dynamicField name="*_p"  type="location" indexed="true" stored="true"/>
 
    <!-- some trie-coded dynamic fields for faster range queries -->
@@ -258,8 +275,8 @@
    
 
    <!-- needed by dedup config -->
-   <dynamicField name="*_sS" type="string"  indexed="false" stored="true"/>
    <field name="signatureField" type="string" indexed="true" stored="false"/>
+   <dynamicField name="*_sS" type="string"  indexed="false" stored="true"/>
 
 
  </fields>
@@ -695,7 +712,7 @@
 
     <!-- since fields of this type are by default not stored or indexed,
          any data added to them will be ignored outright.  --> 
-    <fieldtype name="ignored" stored="false" indexed="false" multiValued="true" class="solr.StrField"
/>
+    <fieldtype name="ignored" stored="false" indexed="false" docValues="false" multiValued="true"
class="solr.StrField" />
 
     <!-- This point type indexes the coordinates as separate fields (subFields)
       If subFieldType is defined, it references a type, and a dynamic field

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6f60ac21/solr/core/src/test/org/apache/solr/search/TestRealTimeGet.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/TestRealTimeGet.java b/solr/core/src/test/org/apache/solr/search/TestRealTimeGet.java
index 3f30440..fe02108 100644
--- a/solr/core/src/test/org/apache/solr/search/TestRealTimeGet.java
+++ b/solr/core/src/test/org/apache/solr/search/TestRealTimeGet.java
@@ -43,7 +43,7 @@ public class TestRealTimeGet extends TestRTGBase {
 
   @BeforeClass
   public static void beforeClass() throws Exception {
-    initCore("solrconfig-tlog.xml","schema15.xml");
+    initCore("solrconfig-tlog.xml","schema_latest.xml");
   }
 
 
@@ -52,12 +52,22 @@ public class TestRealTimeGet extends TestRTGBase {
     clearIndex();
     assertU(commit());
 
-    assertU(adoc("id","1"));
+    assertU(adoc("id","1",
+        "a_f","-1.5", "a_fd","-1.5", "a_fdS","-1.5",                        "a_fs","1.0","a_fs","2.5",
"a_fds","1.0","a_fds","2.5",  "a_fdsS","1.0","a_fdsS","2.5",
+        "a_d","-1.2E99", "a_dd","-1.2E99", "a_ddS","-1.2E99",               "a_ds","1.0","a_ds","2.5",
"a_dds","1.0","a_dds","2.5",  "a_ddsS","1.0","a_ddsS","2.5",
+        "a_i","-1", "a_id","-1", "a_idS","-1",                              "a_is","1","a_is","2",
    "a_ids","1","a_ids","2",      "a_idsS","1","a_idsS","2",
+        "a_l","-9999999999", "a_ld","-9999999999", "a_ldS","-9999999999",   "a_ls","1","a_ls","9999999999",
    "a_lds","1","a_lds","9999999999",      "a_ldsS","1","a_ldsS","9999999999"
+    ));
     assertJQ(req("q","id:1")
         ,"/response/numFound==0"
     );
-    assertJQ(req("qt","/get", "id","1", "fl","id")
-        ,"=={'doc':{'id':'1'}}"
+    assertJQ(req("qt","/get", "id","1", "fl","id, a_f,a_fd,a_fdS   a_fs,a_fds,a_fdsS,  a_d,a_dd,a_ddS,
 a_ds,a_dds,a_ddsS,  a_i,a_id,a_idS   a_is,a_ids,a_idsS,   a_l,a_ld,a_ldS   a_ls,a_lds,a_ldsS")
+        ,"=={'doc':{'id':'1'" +
+            ", a_f:-1.5, a_fd:-1.5, a_fdS:-1.5,  a_fs:[1.0,2.5],      a_fds:[1.0,2.5],a_fdsS:[1.0,2.5]"
+
+            ", a_d:-1.2E99, a_dd:-1.2E99, a_ddS:-1.2E99,              a_ds:[1.0,2.5],a_dds:[1.0,2.5],a_ddsS:[1.0,2.5]"
+
+            ", a_i:-1, a_id:-1, a_idS:-1,                             a_is:[1,2],a_ids:[1,2],a_idsS:[1,2]"
+
+            ", a_l:-9999999999, a_ld:-9999999999, a_ldS:-9999999999,  a_ls:[1,9999999999],a_lds:[1,9999999999],a_ldsS:[1,9999999999]"
+
+            "       }}"
     );
     assertJQ(req("qt","/get","ids","1", "fl","id")
         ,"=={" +
@@ -72,6 +82,17 @@ public class TestRealTimeGet extends TestRTGBase {
     assertJQ(req("q","id:1")
         ,"/response/numFound==1"
     );
+
+    // a cut-n-paste of the first big query, but this time it will be retrieved from the
index rather than the transaction log
+    assertJQ(req("qt","/get", "id","1", "fl","id, a_f,a_fd,a_fdS   a_fs,a_fds,a_fdsS,  a_d,a_dd,a_ddS,
 a_ds,a_dds,a_ddsS,  a_i,a_id,a_idS   a_is,a_ids,a_idsS,   a_l,a_ld,a_ldS   a_ls,a_lds,a_ldsS")
+        ,"=={'doc':{'id':'1'" +
+            ", a_f:-1.5, a_fd:-1.5, a_fdS:-1.5,  a_fs:[1.0,2.5],      a_fds:[1.0,2.5],a_fdsS:[1.0,2.5]"
+
+            ", a_d:-1.2E99, a_dd:-1.2E99, a_ddS:-1.2E99,              a_ds:[1.0,2.5],a_dds:[1.0,2.5],a_ddsS:[1.0,2.5]"
+
+            ", a_i:-1, a_id:-1, a_idS:-1,                             a_is:[1,2],a_ids:[1,2],a_idsS:[1,2]"
+
+            ", a_l:-9999999999, a_ld:-9999999999, a_ldS:-9999999999,  a_ls:[1,9999999999],a_lds:[1,9999999999],a_ldsS:[1,9999999999]"
+
+            "       }}"
+    );
+
     assertJQ(req("qt","/get","id","1", "fl","id")
         ,"=={'doc':{'id':'1'}}"
     );
@@ -101,7 +122,7 @@ public class TestRealTimeGet extends TestRTGBase {
     assertJQ(req("qt","/get","id","10", "fl","id")
         ,"=={'doc':{'id':'10'}}"
     );
-    assertU(delQ("id:10 abcdef"));
+    assertU(delQ("id:10 foo_s:abcdef"));
     assertJQ(req("qt","/get","id","10")
         ,"=={'doc':null}"
     );
@@ -109,6 +130,24 @@ public class TestRealTimeGet extends TestRTGBase {
         ,"=={'doc':{'id':'11'}}"
     );
 
+    // multivalued field
+    assertU(adoc("id","12", "val_ls","1", "val_ls","2"));
+    assertJQ(req("q","id:12")
+        ,"/response/numFound==0"
+    );
+    assertJQ(req("qt","/get", "id","12", "fl","id,val_ls")
+        ,"=={'doc':{'id':'12', 'val_ls':[1,2]}}"
+    );
+
+    assertU(commit());
+
+    assertJQ(req("qt","/get", "id","12", "fl","id,val_ls")
+        ,"=={'doc':{'id':'12', 'val_ls':[1,2]}}"
+    );
+    assertJQ(req("q","id:12")
+        ,"/response/numFound==1"
+    );
+
 
     SolrQueryRequest req = req();
     RefCounted<SolrIndexSearcher> realtimeHolder = req.getCore().getRealtimeSearcher();


Mime
View raw message