lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From va...@apache.org
Subject lucene-solr:branch_7x: SOLR-12616: Optimize Export writer upto 4 sort fields to get better performance. This was removed in SOLR-11598 but brought back in the same version
Date Wed, 08 Aug 2018 20:30:13 GMT
Repository: lucene-solr
Updated Branches:
  refs/heads/branch_7x 3102c21e0 -> 13b9e28f9


SOLR-12616: Optimize Export writer upto 4 sort fields to get better performance. This was
removed in SOLR-11598 but brought back in the same version

(cherry picked from commit e9f3a3c)


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/13b9e28f
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/13b9e28f
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/13b9e28f

Branch: refs/heads/branch_7x
Commit: 13b9e28f9dbb0d117d8758c37d8df7d4c17a9edc
Parents: 3102c21
Author: Varun Thacker <varun@apache.org>
Authored: Wed Aug 8 11:10:21 2018 -0700
Committer: Varun Thacker <varun@apache.org>
Committed: Wed Aug 8 13:29:56 2018 -0700

----------------------------------------------------------------------
 solr/CHANGES.txt                                |   3 +
 .../solr/handler/export/DoubleValueSortDoc.java |  93 ++++++++++++++
 .../solr/handler/export/ExportWriter.java       |  13 +-
 .../solr/handler/export/QuadValueSortDoc.java   | 126 +++++++++++++++++++
 .../solr/handler/export/SingleValueSortDoc.java |  82 ++++++++++++
 .../org/apache/solr/handler/export/SortDoc.java |   7 +-
 .../solr/handler/export/TripleValueSortDoc.java | 110 ++++++++++++++++
 7 files changed, 431 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/13b9e28f/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 68c8155..dbb5289 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -211,6 +211,9 @@ Optimizations
 
 * SOLR-11881: Retry update requests sent by leaders to it's followers (Varun Thacker, Mark
Miller, Tomás Fernández Löbbe)
 
+* SOLR-12616: Optimize Export writer upto 4 sort fields to get better performance.
+  This was removed in SOLR-11598 but brought back in the same version (Amrit Sarkar, Varun
Thacker)
+
 Other Changes
 ----------------------
 

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/13b9e28f/solr/core/src/java/org/apache/solr/handler/export/DoubleValueSortDoc.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/export/DoubleValueSortDoc.java b/solr/core/src/java/org/apache/solr/handler/export/DoubleValueSortDoc.java
new file mode 100644
index 0000000..30fff33
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/export/DoubleValueSortDoc.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.export;
+
+import java.io.IOException;
+
+import org.apache.lucene.index.LeafReaderContext;
+
+class DoubleValueSortDoc extends SingleValueSortDoc {
+
+  protected SortValue value2;
+
+  public void setNextReader(LeafReaderContext context) throws IOException {
+    this.ord = context.ord;
+    this.docBase = context.docBase;
+    value1.setNextReader(context);
+    value2.setNextReader(context);
+  }
+
+  public void reset() {
+    this.docId = -1;
+    this.docBase = -1;
+    value1.reset();
+    value2.reset();
+  }
+
+  public void setValues(int docId) throws IOException {
+    this.docId = docId;
+    value1.setCurrentValue(docId);
+    value2.setCurrentValue(docId);
+  }
+
+  public void setValues(SortDoc sortDoc) {
+    this.docId = sortDoc.docId;
+    this.ord = sortDoc.ord;
+    this.docBase = sortDoc.docBase;
+    value1.setCurrentValue(((DoubleValueSortDoc)sortDoc).value1);
+    value2.setCurrentValue(((DoubleValueSortDoc)sortDoc).value2);
+  }
+
+  public DoubleValueSortDoc(SortValue value1, SortValue value2) {
+    super(value1);
+    this.value2 = value2;
+  }
+
+  public SortDoc copy() {
+    return new DoubleValueSortDoc(value1.copy(), value2.copy());
+  }
+
+  public boolean lessThan(Object o) {
+    DoubleValueSortDoc sd = (DoubleValueSortDoc)o;
+    int comp = value1.compareTo(sd.value1);
+    if(comp == -1) {
+      return true;
+    } else if (comp == 1) {
+      return false;
+    } else {
+      comp = value2.compareTo(sd.value2);
+      if(comp == -1) {
+        return true;
+      } else if (comp == 1) {
+        return false;
+      } else {
+        return docId+docBase > sd.docId+sd.docBase;
+      }
+    }
+  }
+
+  public int compareTo(Object o) {
+    DoubleValueSortDoc sd = (DoubleValueSortDoc)o;
+    int comp = value1.compareTo(sd.value1);
+    if (comp == 0) {
+      return value2.compareTo(sd.value2);
+    } else {
+      return comp;
+    }
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/13b9e28f/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java b/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java
index 57ce600..9cacfcc 100644
--- a/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java
+++ b/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java
@@ -431,7 +431,18 @@ public class ExportWriter implements SolrCore.RawWriter, Closeable {
         throw new IOException("Sort fields must be one of the following types: int,float,long,double,string,date,boolean");
       }
     }
-
+    //SingleValueSortDoc etc are specialized classes which don't have array lookups. On benchmarking
large datasets
+    //This is faster than the using an array in SortDoc . So upto 4 sort fields we still
want to keep specialized classes.
+    //SOLR-12616 has more details
+    if (sortValues.length == 1) {
+      return new SingleValueSortDoc(sortValues[0]);
+    } else if (sortValues.length == 2) {
+      return new DoubleValueSortDoc(sortValues[0], sortValues[1]);
+    } else if (sortValues.length == 3) {
+      return new TripleValueSortDoc(sortValues[0], sortValues[1], sortValues[2]);
+    } else if (sortValues.length == 4) {
+      return new QuadValueSortDoc(sortValues[0], sortValues[1], sortValues[2], sortValues[3]);
+    }
     return new SortDoc(sortValues);
   }
 

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/13b9e28f/solr/core/src/java/org/apache/solr/handler/export/QuadValueSortDoc.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/export/QuadValueSortDoc.java b/solr/core/src/java/org/apache/solr/handler/export/QuadValueSortDoc.java
new file mode 100644
index 0000000..c2dfcba
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/export/QuadValueSortDoc.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.export;
+
+import java.io.IOException;
+
+import org.apache.lucene.index.LeafReaderContext;
+
+class QuadValueSortDoc extends TripleValueSortDoc {
+
+  protected SortValue value4;
+
+  public void setNextReader(LeafReaderContext context) throws IOException {
+    this.ord = context.ord;
+    this.docBase = context.docBase;
+    value1.setNextReader(context);
+    value2.setNextReader(context);
+    value3.setNextReader(context);
+    value4.setNextReader(context);
+  }
+
+  public void reset() {
+    this.docId = -1;
+    this.docBase = -1;
+    value1.reset();
+    value2.reset();
+    value3.reset();
+    value4.reset();
+  }
+
+  public void setValues(int docId) throws IOException {
+    this.docId = docId;
+    value1.setCurrentValue(docId);
+    value2.setCurrentValue(docId);
+    value3.setCurrentValue(docId);
+    value4.setCurrentValue(docId);
+  }
+
+  public void setValues(SortDoc sortDoc) {
+    this.docId = sortDoc.docId;
+    this.ord = sortDoc.ord;
+    this.docBase = sortDoc.docBase;
+    value1.setCurrentValue(((QuadValueSortDoc)sortDoc).value1);
+    value2.setCurrentValue(((QuadValueSortDoc)sortDoc).value2);
+    value3.setCurrentValue(((QuadValueSortDoc)sortDoc).value3);
+    value4.setCurrentValue(((QuadValueSortDoc)sortDoc).value4);
+  }
+
+  public QuadValueSortDoc(SortValue value1, SortValue value2, SortValue value3, SortValue
value4) {
+    super(value1, value2, value3);
+    this.value4 = value4;
+  }
+
+  public SortDoc copy() {
+    return new QuadValueSortDoc(value1.copy(), value2.copy(), value3.copy(), value4.copy());
+  }
+
+  public boolean lessThan(Object o) {
+
+    QuadValueSortDoc sd = (QuadValueSortDoc)o;
+    int comp = value1.compareTo(sd.value1);
+    if(comp == -1) {
+      return true;
+    } else if (comp == 1) {
+      return false;
+    } else {
+      comp = value2.compareTo(sd.value2);
+      if(comp == -1) {
+        return true;
+      } else if (comp == 1) {
+        return false;
+      } else {
+        comp = value3.compareTo(sd.value3);
+        if(comp == -1) {
+          return true;
+        } else if (comp == 1) {
+          return false;
+        } else {
+          comp = value4.compareTo(sd.value4);
+          if(comp == -1) {
+            return true;
+          } else if (comp == 1) {
+            return false;
+          } else {
+            return docId+docBase > sd.docId+sd.docBase;
+          }
+        }
+      }
+    }
+  }
+
+  public int compareTo(Object o) {
+    QuadValueSortDoc sd = (QuadValueSortDoc)o;
+    int comp = value1.compareTo(sd.value1);
+    if(comp == 0) {
+      comp = value2.compareTo(sd.value2);
+      if(comp == 0) {
+        comp = value3.compareTo(sd.value3);
+        if(comp == 0) {
+          return value4.compareTo(sd.value4);
+        } else {
+          return comp;
+        }
+      } else {
+        return comp;
+      }
+    } else {
+      return comp;
+    }
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/13b9e28f/solr/core/src/java/org/apache/solr/handler/export/SingleValueSortDoc.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/export/SingleValueSortDoc.java b/solr/core/src/java/org/apache/solr/handler/export/SingleValueSortDoc.java
new file mode 100644
index 0000000..88fc799
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/export/SingleValueSortDoc.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.export;
+
+import java.io.IOException;
+
+import org.apache.lucene.index.LeafReaderContext;
+
+class SingleValueSortDoc extends SortDoc {
+
+  protected SortValue value1;
+
+  public void setNextReader(LeafReaderContext context) throws IOException {
+    this.ord = context.ord;
+    this.docBase = context.docBase;
+    value1.setNextReader(context);
+  }
+
+  public void reset() {
+    this.docId = -1;
+    this.docBase = -1;
+    this.value1.reset();
+  }
+
+  public void setValues(int docId) throws IOException {
+    this.docId = docId;
+    value1.setCurrentValue(docId);
+  }
+
+  public void setValues(SortDoc sortDoc) {
+    this.docId = sortDoc.docId;
+    this.ord = sortDoc.ord;
+    this.docBase = sortDoc.docBase;
+    value1.setCurrentValue(((SingleValueSortDoc)sortDoc).value1);
+  }
+
+  public SingleValueSortDoc(SortValue value1) {
+    super();
+    this.value1 = value1;
+  }
+
+  public SortDoc copy() {
+    return new SingleValueSortDoc(value1.copy());
+  }
+
+  public boolean lessThan(Object o) {
+    SingleValueSortDoc sd = (SingleValueSortDoc)o;
+    int comp = value1.compareTo(sd.value1);
+    if(comp == -1) {
+      return true;
+    } else if (comp == 1) {
+      return false;
+    } else {
+      return docId+docBase > sd.docId+sd.docBase;
+    }
+  }
+
+  public int compareTo(Object o) {
+    SingleValueSortDoc sd = (SingleValueSortDoc)o;
+    return value1.compareTo(sd.value1);
+  }
+
+  public String toString() {
+    return docId+":"+value1.toString();
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/13b9e28f/solr/core/src/java/org/apache/solr/handler/export/SortDoc.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/export/SortDoc.java b/solr/core/src/java/org/apache/solr/handler/export/SortDoc.java
index 3cf478b..1302332 100644
--- a/solr/core/src/java/org/apache/solr/handler/export/SortDoc.java
+++ b/solr/core/src/java/org/apache/solr/handler/export/SortDoc.java
@@ -32,6 +32,8 @@ class SortDoc {
   public SortDoc(SortValue[] sortValues) {
     this.sortValues = sortValues;
   }
+  public SortDoc() {
+  }
 
   public void setNextReader(LeafReaderContext context) throws IOException {
     this.ord = context.ord;
@@ -43,6 +45,7 @@ class SortDoc {
 
   public void reset() {
     this.docId = -1;
+    this.docBase = -1;
     for (SortValue value : sortValues) {
       value.reset();
     }
@@ -82,9 +85,9 @@ class SortDoc {
     SortValue[] sortValues1 = sd.sortValues;
     for(int i=0; i<sortValues.length; i++) {
       int comp = sortValues[i].compareTo(sortValues1[i]);
-      if(comp < 0) {
+      if (comp < 0) {
         return true;
-      } if(comp > 0) {
+      } else if (comp > 0) {
         return false;
       }
     }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/13b9e28f/solr/core/src/java/org/apache/solr/handler/export/TripleValueSortDoc.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/export/TripleValueSortDoc.java b/solr/core/src/java/org/apache/solr/handler/export/TripleValueSortDoc.java
new file mode 100644
index 0000000..0241ebc
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/export/TripleValueSortDoc.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.export;
+
+import java.io.IOException;
+
+import org.apache.lucene.index.LeafReaderContext;
+
+class TripleValueSortDoc extends DoubleValueSortDoc {
+
+  protected SortValue value3;
+
+  public void setNextReader(LeafReaderContext context) throws IOException {
+    this.ord = context.ord;
+    this.docBase = context.docBase;
+    value1.setNextReader(context);
+    value2.setNextReader(context);
+    value3.setNextReader(context);
+  }
+
+  public void reset() {
+    this.docId = -1;
+    this.docBase = -1;
+    value1.reset();
+    value2.reset();
+    value3.reset();
+  }
+
+  public void setValues(int docId) throws IOException {
+    this.docId = docId;
+    value1.setCurrentValue(docId);
+    value2.setCurrentValue(docId);
+    value3.setCurrentValue(docId);
+  }
+
+  public void setValues(SortDoc sortDoc) {
+    this.docId = sortDoc.docId;
+    this.ord = sortDoc.ord;
+    this.docBase = sortDoc.docBase;
+    value1.setCurrentValue(((TripleValueSortDoc)sortDoc).value1);
+    value2.setCurrentValue(((TripleValueSortDoc)sortDoc).value2);
+    value3.setCurrentValue(((TripleValueSortDoc)sortDoc).value3);
+  }
+
+  public TripleValueSortDoc(SortValue value1, SortValue value2, SortValue value3) {
+    super(value1, value2);
+    this.value3 = value3;
+  }
+
+  public SortDoc copy() {
+    return new TripleValueSortDoc(value1.copy(), value2.copy(), value3.copy());
+  }
+
+  public boolean lessThan(Object o) {
+
+    TripleValueSortDoc sd = (TripleValueSortDoc)o;
+    int comp = value1.compareTo(sd.value1);
+    if(comp == -1) {
+      return true;
+    } else if (comp == 1) {
+      return false;
+    } else {
+      comp = value2.compareTo(sd.value2);
+      if(comp == -1) {
+        return true;
+      } else if (comp == 1) {
+        return false;
+      } else {
+        comp = value3.compareTo(sd.value3);
+        if(comp == -1) {
+          return true;
+        } else if (comp == 1) {
+          return false;
+        } else {
+          return docId+docBase > sd.docId+sd.docBase;
+        }
+      }
+    }
+  }
+
+  public int compareTo(Object o) {
+    TripleValueSortDoc sd = (TripleValueSortDoc)o;
+    int comp = value1.compareTo(sd.value1);
+    if (comp == 0) {
+      comp = value2.compareTo(sd.value2);
+      if (comp == 0) {
+        return value3.compareTo(sd.value3);
+      } else {
+        return comp;
+      }
+    } else {
+      return comp;
+    }
+  }
+}


Mime
View raw message