Return-Path: X-Original-To: apmail-lucene-commits-archive@www.apache.org Delivered-To: apmail-lucene-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 03835FB61 for ; Fri, 10 May 2013 07:01:58 +0000 (UTC) Received: (qmail 33888 invoked by uid 500); 10 May 2013 07:01:57 -0000 Mailing-List: contact commits-help@lucene.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@lucene.apache.org Delivered-To: mailing list commits@lucene.apache.org Received: (qmail 33865 invoked by uid 99); 10 May 2013 07:01:57 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 10 May 2013 07:01:57 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 10 May 2013 07:01:52 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id AFAE12388847; Fri, 10 May 2013 07:01:30 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1480897 - in /lucene/dev/branches/lucene_solr_4_3: ./ dev-tools/ lucene/ lucene/analysis/ lucene/analysis/icu/src/java/org/apache/lucene/collation/ lucene/backwards/ lucene/benchmark/ lucene/classification/ lucene/classification/src/ lucen... Date: Fri, 10 May 2013 07:01:30 -0000 To: commits@lucene.apache.org From: shalin@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20130510070130.AFAE12388847@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: shalin Date: Fri May 10 07:01:28 2013 New Revision: 1480897 URL: http://svn.apache.org/r1480897 Log: SOLR-4798: use correct router during index splitting Modified: lucene/dev/branches/lucene_solr_4_3/ (props changed) lucene/dev/branches/lucene_solr_4_3/dev-tools/ (props changed) lucene/dev/branches/lucene_solr_4_3/lucene/ (props changed) lucene/dev/branches/lucene_solr_4_3/lucene/BUILD.txt (props changed) lucene/dev/branches/lucene_solr_4_3/lucene/CHANGES.txt (props changed) lucene/dev/branches/lucene_solr_4_3/lucene/JRE_VERSION_MIGRATION.txt (props changed) lucene/dev/branches/lucene_solr_4_3/lucene/LICENSE.txt (props changed) lucene/dev/branches/lucene_solr_4_3/lucene/MIGRATE.txt (props changed) lucene/dev/branches/lucene_solr_4_3/lucene/NOTICE.txt (props changed) lucene/dev/branches/lucene_solr_4_3/lucene/README.txt (props changed) lucene/dev/branches/lucene_solr_4_3/lucene/SYSTEM_REQUIREMENTS.txt (props changed) lucene/dev/branches/lucene_solr_4_3/lucene/analysis/ (props changed) lucene/dev/branches/lucene_solr_4_3/lucene/analysis/icu/src/java/org/apache/lucene/collation/ICUCollationKeyFilterFactory.java (props changed) lucene/dev/branches/lucene_solr_4_3/lucene/backwards/ (props changed) lucene/dev/branches/lucene_solr_4_3/lucene/benchmark/ (props changed) lucene/dev/branches/lucene_solr_4_3/lucene/build.xml (props changed) lucene/dev/branches/lucene_solr_4_3/lucene/classification/ (props changed) lucene/dev/branches/lucene_solr_4_3/lucene/classification/build.xml (props changed) lucene/dev/branches/lucene_solr_4_3/lucene/classification/ivy.xml (props changed) lucene/dev/branches/lucene_solr_4_3/lucene/classification/src/ (props changed) lucene/dev/branches/lucene_solr_4_3/lucene/codecs/ (props changed) lucene/dev/branches/lucene_solr_4_3/lucene/common-build.xml (props changed) lucene/dev/branches/lucene_solr_4_3/lucene/core/ (props changed) lucene/dev/branches/lucene_solr_4_3/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (props changed) lucene/dev/branches/lucene_solr_4_3/lucene/core/src/test/org/apache/lucene/index/index.40.cfs.zip (props changed) lucene/dev/branches/lucene_solr_4_3/lucene/core/src/test/org/apache/lucene/index/index.40.nocfs.zip (props changed) lucene/dev/branches/lucene_solr_4_3/lucene/core/src/test/org/apache/lucene/index/index.40.optimized.cfs.zip (props changed) lucene/dev/branches/lucene_solr_4_3/lucene/core/src/test/org/apache/lucene/index/index.40.optimized.nocfs.zip (props changed) lucene/dev/branches/lucene_solr_4_3/lucene/core/src/test/org/apache/lucene/search/TestSort.java (props changed) lucene/dev/branches/lucene_solr_4_3/lucene/core/src/test/org/apache/lucene/search/TestSortDocValues.java (props changed) lucene/dev/branches/lucene_solr_4_3/lucene/core/src/test/org/apache/lucene/search/TestSortRandom.java (props changed) lucene/dev/branches/lucene_solr_4_3/lucene/core/src/test/org/apache/lucene/search/TestTopFieldCollector.java (props changed) lucene/dev/branches/lucene_solr_4_3/lucene/core/src/test/org/apache/lucene/search/TestTotalHitCountCollector.java (props changed) lucene/dev/branches/lucene_solr_4_3/lucene/demo/ (props changed) lucene/dev/branches/lucene_solr_4_3/lucene/facet/ (props changed) lucene/dev/branches/lucene_solr_4_3/lucene/grouping/ (props changed) lucene/dev/branches/lucene_solr_4_3/lucene/highlighter/ (props changed) lucene/dev/branches/lucene_solr_4_3/lucene/ivy-settings.xml (props changed) lucene/dev/branches/lucene_solr_4_3/lucene/join/ (props changed) lucene/dev/branches/lucene_solr_4_3/lucene/licenses/ (props changed) lucene/dev/branches/lucene_solr_4_3/lucene/memory/ (props changed) lucene/dev/branches/lucene_solr_4_3/lucene/misc/ (props changed) lucene/dev/branches/lucene_solr_4_3/lucene/module-build.xml (props changed) lucene/dev/branches/lucene_solr_4_3/lucene/queries/ (props changed) lucene/dev/branches/lucene_solr_4_3/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionQuerySort.java (props changed) lucene/dev/branches/lucene_solr_4_3/lucene/queryparser/ (props changed) lucene/dev/branches/lucene_solr_4_3/lucene/sandbox/ (props changed) lucene/dev/branches/lucene_solr_4_3/lucene/site/ (props changed) lucene/dev/branches/lucene_solr_4_3/lucene/spatial/ (props changed) lucene/dev/branches/lucene_solr_4_3/lucene/suggest/ (props changed) lucene/dev/branches/lucene_solr_4_3/lucene/test-framework/ (props changed) lucene/dev/branches/lucene_solr_4_3/lucene/tools/ (props changed) lucene/dev/branches/lucene_solr_4_3/solr/ (props changed) lucene/dev/branches/lucene_solr_4_3/solr/CHANGES.txt (contents, props changed) lucene/dev/branches/lucene_solr_4_3/solr/LICENSE.txt (props changed) lucene/dev/branches/lucene_solr_4_3/solr/NOTICE.txt (props changed) lucene/dev/branches/lucene_solr_4_3/solr/README.txt (props changed) lucene/dev/branches/lucene_solr_4_3/solr/SYSTEM_REQUIREMENTS.txt (props changed) lucene/dev/branches/lucene_solr_4_3/solr/build.xml (props changed) lucene/dev/branches/lucene_solr_4_3/solr/cloud-dev/ (props changed) lucene/dev/branches/lucene_solr_4_3/solr/common-build.xml (props changed) lucene/dev/branches/lucene_solr_4_3/solr/contrib/ (props changed) lucene/dev/branches/lucene_solr_4_3/solr/core/ (props changed) lucene/dev/branches/lucene_solr_4_3/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java lucene/dev/branches/lucene_solr_4_3/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java lucene/dev/branches/lucene_solr_4_3/solr/core/src/java/org/apache/solr/update/SplitIndexCommand.java lucene/dev/branches/lucene_solr_4_3/solr/core/src/test/org/apache/solr/core/TestConfig.java (props changed) lucene/dev/branches/lucene_solr_4_3/solr/core/src/test/org/apache/solr/update/SolrIndexSplitterTest.java lucene/dev/branches/lucene_solr_4_3/solr/example/ (props changed) lucene/dev/branches/lucene_solr_4_3/solr/licenses/ (props changed) lucene/dev/branches/lucene_solr_4_3/solr/licenses/httpclient-LICENSE-ASL.txt (props changed) lucene/dev/branches/lucene_solr_4_3/solr/licenses/httpclient-NOTICE.txt (props changed) lucene/dev/branches/lucene_solr_4_3/solr/licenses/httpcore-LICENSE-ASL.txt (props changed) lucene/dev/branches/lucene_solr_4_3/solr/licenses/httpcore-NOTICE.txt (props changed) lucene/dev/branches/lucene_solr_4_3/solr/licenses/httpmime-LICENSE-ASL.txt (props changed) lucene/dev/branches/lucene_solr_4_3/solr/licenses/httpmime-NOTICE.txt (props changed) lucene/dev/branches/lucene_solr_4_3/solr/scripts/ (props changed) lucene/dev/branches/lucene_solr_4_3/solr/site/ (props changed) lucene/dev/branches/lucene_solr_4_3/solr/solrj/ (props changed) lucene/dev/branches/lucene_solr_4_3/solr/solrj/src/java/org/apache/solr/common/cloud/CompositeIdRouter.java lucene/dev/branches/lucene_solr_4_3/solr/solrj/src/java/org/apache/solr/common/cloud/HashBasedRouter.java lucene/dev/branches/lucene_solr_4_3/solr/test-framework/ (props changed) lucene/dev/branches/lucene_solr_4_3/solr/webapp/ (props changed) Modified: lucene/dev/branches/lucene_solr_4_3/solr/CHANGES.txt URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_3/solr/CHANGES.txt?rev=1480897&r1=1480896&r2=1480897&view=diff ============================================================================== --- lucene/dev/branches/lucene_solr_4_3/solr/CHANGES.txt (original) +++ lucene/dev/branches/lucene_solr_4_3/solr/CHANGES.txt Fri May 10 07:01:28 2013 @@ -40,6 +40,13 @@ Bug Fixes * SOLR-4795: Sub shard leader should not accept any updates from parent after it goes active (shalin) +* SOLR-4798: shard splitting does not respect the router for the collection + when executing the index split. One effect of this is that documents + may be placed in the wrong shard when the default compositeId router + is used in conjunction with IDs containing "!". (yonik) + + + Other Changes ---------------------- Modified: lucene/dev/branches/lucene_solr_4_3/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_3/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java?rev=1480897&r1=1480896&r2=1480897&view=diff ============================================================================== --- lucene/dev/branches/lucene_solr_4_3/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java (original) +++ lucene/dev/branches/lucene_solr_4_3/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java Fri May 10 07:01:28 2013 @@ -252,6 +252,7 @@ public class CoreAdminHandler extends Re List paths = null; int partitions = pathsArr != null ? pathsArr.length : newCoreNames.length; + DocRouter router = null; if (coreContainer.isZooKeeperAware()) { ClusterState clusterState = coreContainer.getZkController().getClusterState(); String collectionName = req.getCore().getCoreDescriptor().getCloudDescriptor().getCollectionName(); @@ -259,8 +260,8 @@ public class CoreAdminHandler extends Re String sliceName = req.getCore().getCoreDescriptor().getCloudDescriptor().getShardId(); Slice slice = clusterState.getSlice(collectionName, sliceName); DocRouter.Range currentRange = slice.getRange(); - DocRouter hp = collection.getRouter() != null ? collection.getRouter() : DocRouter.DEFAULT; - ranges = currentRange != null ? hp.partitionRange(partitions, currentRange) : null; + router = collection.getRouter() != null ? collection.getRouter() : DocRouter.DEFAULT; + ranges = currentRange != null ? router.partitionRange(partitions, currentRange) : null; } if (pathsArr == null) { @@ -278,7 +279,7 @@ public class CoreAdminHandler extends Re } - SplitIndexCommand cmd = new SplitIndexCommand(req, paths, newCores, ranges); + SplitIndexCommand cmd = new SplitIndexCommand(req, paths, newCores, ranges, router); core.getUpdateHandler().split(cmd); // After the split has completed, someone (here?) should start the process of replaying the buffered updates. Modified: lucene/dev/branches/lucene_solr_4_3/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_3/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java?rev=1480897&r1=1480896&r2=1480897&view=diff ============================================================================== --- lucene/dev/branches/lucene_solr_4_3/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java (original) +++ lucene/dev/branches/lucene_solr_4_3/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java Fri May 10 07:01:28 2013 @@ -32,9 +32,11 @@ import org.apache.lucene.util.CharsRef; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.OpenBitSet; import org.apache.solr.common.cloud.DocRouter; +import org.apache.solr.common.cloud.HashBasedRouter; import org.apache.solr.common.util.Hash; import org.apache.solr.core.SolrCore; import org.apache.solr.schema.SchemaField; +import org.apache.solr.schema.StrField; import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.util.RefCounted; import org.slf4j.Logger; @@ -53,6 +55,8 @@ public class SolrIndexSplitter { DocRouter.Range[] rangesArr; // same as ranges list, but an array for extra speed in inner loops List paths; List cores; + DocRouter router; + HashBasedRouter hashRouter; int numPieces; int currPartition = 0; @@ -62,6 +66,9 @@ public class SolrIndexSplitter { ranges = cmd.ranges; paths = cmd.paths; cores = cmd.cores; + router = cmd.router; + hashRouter = router instanceof HashBasedRouter ? (HashBasedRouter)router : null; + if (ranges == null) { numPieces = paths != null ? paths.size() : cores.size(); } else { @@ -151,16 +158,24 @@ public class SolrIndexSplitter { BytesRef term = null; DocsEnum docsEnum = null; + CharsRef idRef = new CharsRef(100); for (;;) { term = termsEnum.next(); if (term == null) break; // figure out the hash for the term - // TODO: hook in custom hashes (or store hashes) - // TODO: performance implications of using indexedToReadable? - CharsRef ref = new CharsRef(term.length); - ref = field.getType().indexedToReadable(term, ref); - int hash = Hash.murmurhash3_x86_32(ref, ref.offset, ref.length, 0); + + // FUTURE: if conversion to strings costs too much, we could + // specialize and use the hash function that can work over bytes. + idRef = field.getType().indexedToReadable(term, idRef); + String idString = idRef.toString(); + + int hash = 0; + if (hashRouter != null) { + hash = hashRouter.sliceHash(idString, null, null); + } + // int hash = Hash.murmurhash3_x86_32(ref, ref.offset, ref.length, 0); + docsEnum = termsEnum.docs(liveDocs, docsEnum, DocsEnum.FLAG_NONE); for (;;) { int doc = docsEnum.nextDoc(); Modified: lucene/dev/branches/lucene_solr_4_3/solr/core/src/java/org/apache/solr/update/SplitIndexCommand.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_3/solr/core/src/java/org/apache/solr/update/SplitIndexCommand.java?rev=1480897&r1=1480896&r2=1480897&view=diff ============================================================================== --- lucene/dev/branches/lucene_solr_4_3/solr/core/src/java/org/apache/solr/update/SplitIndexCommand.java (original) +++ lucene/dev/branches/lucene_solr_4_3/solr/core/src/java/org/apache/solr/update/SplitIndexCommand.java Fri May 10 07:01:28 2013 @@ -34,13 +34,14 @@ public class SplitIndexCommand extends U public List paths; public List cores; // either paths or cores should be specified public List ranges; - // TODO: allow specification of custom hash function + public DocRouter router; - public SplitIndexCommand(SolrQueryRequest req, List paths, List cores, List ranges) { + public SplitIndexCommand(SolrQueryRequest req, List paths, List cores, List ranges, DocRouter router) { super(req); this.paths = paths; this.cores = cores; this.ranges = ranges; + this.router = router; } @Override @@ -54,6 +55,7 @@ public class SplitIndexCommand extends U sb.append(",paths=" + paths); sb.append(",cores=" + cores); sb.append(",ranges=" + ranges); + sb.append(",router=" + router); sb.append('}'); return sb.toString(); } Modified: lucene/dev/branches/lucene_solr_4_3/solr/core/src/test/org/apache/solr/update/SolrIndexSplitterTest.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_3/solr/core/src/test/org/apache/solr/update/SolrIndexSplitterTest.java?rev=1480897&r1=1480896&r2=1480897&view=diff ============================================================================== --- lucene/dev/branches/lucene_solr_4_3/solr/core/src/test/org/apache/solr/update/SolrIndexSplitterTest.java (original) +++ lucene/dev/branches/lucene_solr_4_3/solr/core/src/test/org/apache/solr/update/SolrIndexSplitterTest.java Fri May 10 07:01:28 2013 @@ -95,7 +95,7 @@ public class SolrIndexSplitterTest exten request = lrf.makeRequest("q", "dummy"); SplitIndexCommand command = new SplitIndexCommand(request, - Lists.newArrayList(indexDir1.getAbsolutePath(), indexDir2.getAbsolutePath()), null, ranges); + Lists.newArrayList(indexDir1.getAbsolutePath(), indexDir2.getAbsolutePath()), null, ranges, new PlainIdRouter()); new SolrIndexSplitter(command).split(); Directory directory = h.getCore().getDirectoryFactory().get(indexDir1.getAbsolutePath(), @@ -148,7 +148,7 @@ public class SolrIndexSplitterTest exten try { request = lrf.makeRequest("q", "dummy"); - SplitIndexCommand command = new SplitIndexCommand(request, null, Lists.newArrayList(core1, core2), ranges); + SplitIndexCommand command = new SplitIndexCommand(request, null, Lists.newArrayList(core1, core2), ranges, new PlainIdRouter()); new SolrIndexSplitter(command).split(); } finally { if (request != null) request.close(); @@ -185,7 +185,7 @@ public class SolrIndexSplitterTest exten request = lrf.makeRequest("q", "dummy"); SplitIndexCommand command = new SplitIndexCommand(request, - Lists.newArrayList(indexDir1.getAbsolutePath(), indexDir2.getAbsolutePath(), indexDir3.getAbsolutePath()), null, null); + Lists.newArrayList(indexDir1.getAbsolutePath(), indexDir2.getAbsolutePath(), indexDir3.getAbsolutePath()), null, null, new PlainIdRouter()); new SolrIndexSplitter(command).split(); directory = h.getCore().getDirectoryFactory().get(indexDir1.getAbsolutePath(), Modified: lucene/dev/branches/lucene_solr_4_3/solr/solrj/src/java/org/apache/solr/common/cloud/CompositeIdRouter.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_3/solr/solrj/src/java/org/apache/solr/common/cloud/CompositeIdRouter.java?rev=1480897&r1=1480896&r2=1480897&view=diff ============================================================================== --- lucene/dev/branches/lucene_solr_4_3/solr/solrj/src/java/org/apache/solr/common/cloud/CompositeIdRouter.java (original) +++ lucene/dev/branches/lucene_solr_4_3/solr/solrj/src/java/org/apache/solr/common/cloud/CompositeIdRouter.java Fri May 10 07:01:28 2013 @@ -36,7 +36,7 @@ public class CompositeIdRouter extends H private int separator = '!'; // separator used to optionally specify number of bits to allocate toward first part. - private int bitsSepartor = '/'; + private int bitsSeparator = '/'; private int bits = 16; private int mask1 = 0xffff0000; private int mask2 = 0x0000ffff; @@ -59,7 +59,7 @@ public class CompositeIdRouter extends H } @Override - protected int sliceHash(String id, SolrInputDocument doc, SolrParams params) { + public int sliceHash(String id, SolrInputDocument doc, SolrParams params) { int idx = id.indexOf(separator); if (idx < 0) { return Hash.murmurhash3_x86_32(id, 0, id.length(), 0); @@ -69,7 +69,7 @@ public class CompositeIdRouter extends H int m2 = mask2; String part1 = id.substring(0,idx); - int commaIdx = part1.indexOf(bitsSepartor); + int commaIdx = part1.indexOf(bitsSeparator); if (commaIdx > 0) { int firstBits = getBits(part1, commaIdx); if (firstBits >= 0) { @@ -105,7 +105,7 @@ public class CompositeIdRouter extends H int m2 = mask2; String part1 = id.substring(0,idx); - int bitsSepIdx = part1.indexOf(bitsSepartor); + int bitsSepIdx = part1.indexOf(bitsSeparator); if (bitsSepIdx > 0) { int firstBits = getBits(part1, bitsSepIdx); if (firstBits >= 0) { Modified: lucene/dev/branches/lucene_solr_4_3/solr/solrj/src/java/org/apache/solr/common/cloud/HashBasedRouter.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_3/solr/solrj/src/java/org/apache/solr/common/cloud/HashBasedRouter.java?rev=1480897&r1=1480896&r2=1480897&view=diff ============================================================================== --- lucene/dev/branches/lucene_solr_4_3/solr/solrj/src/java/org/apache/solr/common/cloud/HashBasedRouter.java (original) +++ lucene/dev/branches/lucene_solr_4_3/solr/solrj/src/java/org/apache/solr/common/cloud/HashBasedRouter.java Fri May 10 07:01:28 2013 @@ -42,7 +42,7 @@ public abstract class HashBasedRouter ex return range != null && range.includes(hash); } - protected int sliceHash(String id, SolrInputDocument sdoc, SolrParams params) { + public int sliceHash(String id, SolrInputDocument sdoc, SolrParams params) { return Hash.murmurhash3_x86_32(id, 0, id.length(), 0); }