Return-Path: X-Original-To: apmail-lucene-commits-archive@www.apache.org Delivered-To: apmail-lucene-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 586E510873 for ; Wed, 18 Sep 2013 15:14:27 +0000 (UTC) Received: (qmail 22327 invoked by uid 500); 18 Sep 2013 15:14:26 -0000 Mailing-List: contact commits-help@lucene.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@lucene.apache.org Delivered-To: mailing list commits@lucene.apache.org Received: (qmail 22320 invoked by uid 99); 18 Sep 2013 15:14:26 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 18 Sep 2013 15:14:26 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 18 Sep 2013 15:14:20 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 5D95923888A6; Wed, 18 Sep 2013 15:13:58 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1524446 - in /lucene/dev/branches/branch_4x: ./ solr/ solr/core/ solr/core/src/java/org/apache/solr/cloud/ solr/core/src/java/org/apache/solr/handler/admin/ solr/core/src/test/org/apache/solr/cloud/ solr/solrj/ solr/solrj/src/java/org/apac... Date: Wed, 18 Sep 2013 15:13:57 -0000 To: commits@lucene.apache.org From: noble@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20130918151358.5D95923888A6@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: noble Date: Wed Sep 18 15:13:57 2013 New Revision: 1524446 URL: http://svn.apache.org/r1524446 Log: SOLR-4221 changed format router is stored Modified: lucene/dev/branches/branch_4x/ (props changed) lucene/dev/branches/branch_4x/solr/ (props changed) lucene/dev/branches/branch_4x/solr/core/ (props changed) lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/Overseer.java lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/ZkController.java lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/CustomCollectionTest.java lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/SliceStateUpdateTest.java lucene/dev/branches/branch_4x/solr/solrj/ (props changed) lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/CompositeIdRouter.java lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/DocCollection.java lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/DocRouter.java lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ImplicitDocRouter.java lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ZkNodeProps.java lucene/dev/branches/branch_4x/solr/test-framework/ (props changed) lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/Overseer.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/Overseer.java?rev=1524446&r1=1524445&r2=1524446&view=diff ============================================================================== --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/Overseer.java (original) +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/Overseer.java Wed Sep 18 15:13:57 2013 @@ -221,7 +221,7 @@ public class Overseer { ArrayList shardNames = new ArrayList(); - if(ImplicitDocRouter.NAME.equals( message.getStr("router",DocRouter.DEFAULT_NAME))){ + if(ImplicitDocRouter.NAME.equals( message.getStr("router.name",DocRouter.DEFAULT_NAME))){ getShardNames(shardNames,message.getStr("shards",DocRouter.DEFAULT_NAME)); } else { int numShards = message.getInt(ZkStateReader.NUM_SHARDS_PROP, -1); @@ -415,8 +415,8 @@ public class Overseer { private ClusterState createCollection(ClusterState state, String collectionName, List shards , ZkNodeProps message) { log.info("Create collection {} with shards {}", collectionName, shards);; - String routerName = message.getStr(OverseerCollectionProcessor.ROUTER,DocRouter.DEFAULT_NAME); - DocRouter router = DocRouter.getDocRouter(routerName); +// String routerName = message.getStr(OverseerCollectionProcessor.ROUTER,DocRouter.DEFAULT_NAME); + DocRouter router = DocRouter.getDocRouter(message.getStr(OverseerCollectionProcessor.ROUTER,DocRouter.DEFAULT_NAME)); List ranges = router.partitionRange(shards.size(), router.fullRange()); @@ -447,7 +447,7 @@ public class Overseer { } if(val != null) collectionProps.put(e.getKey(),val); } - collectionProps.put(DocCollection.DOC_ROUTER, routerName); + collectionProps.put(DocCollection.DOC_ROUTER, DocRouter.getRouterSpec(message)); DocCollection newCollection = new DocCollection(collectionName, newSlices, collectionProps, router); @@ -506,7 +506,7 @@ public class Overseer { // without explicitly creating a collection. In this current case, we assume custom sharding with an "implicit" router. slices = new HashMap(1); props = new HashMap(1); - props.put(DocCollection.DOC_ROUTER, ImplicitDocRouter.NAME); + props.put(DocCollection.DOC_ROUTER, ZkNodeProps.makeMap("name",ImplicitDocRouter.NAME)); router = new ImplicitDocRouter(); } else { props = coll.getProperties(); Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java?rev=1524446&r1=1524445&r2=1524446&view=diff ============================================================================== --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java (original) +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java Wed Sep 18 15:13:57 2013 @@ -64,7 +64,6 @@ import java.util.Set; import static org.apache.solr.cloud.Assign.Node; import static org.apache.solr.cloud.Assign.getNodesForNewShard; -import static org.apache.solr.common.cloud.DocRouter.ROUTE_FIELD; import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP; @@ -102,11 +101,10 @@ public class OverseerCollectionProcessor public static final String COLL_CONF = "collection.configName"; - public static final Map COLL_PROPS = asMap( - ROUTER,DocRouter.DEFAULT_NAME, + public static final Map COLL_PROPS = ZkNodeProps.makeMap( + ROUTER, DocRouter.DEFAULT_NAME, REPLICATION_FACTOR, "1", - MAX_SHARDS_PER_NODE,"1", - ROUTE_FIELD,null); + MAX_SHARDS_PER_NODE, "1"); // TODO: use from Overseer? @@ -875,7 +873,7 @@ public class OverseerCollectionProcessor int repFactor = message.getInt( REPLICATION_FACTOR, 1); Integer numSlices = message.getInt(NUM_SLICES, null); - String router = message.getStr(ROUTER, DocRouter.DEFAULT_NAME); + String router = message.getStr("router.name", DocRouter.DEFAULT_NAME); List shardNames = new ArrayList(); if(ImplicitDocRouter.NAME.equals(router)){ Overseer.getShardNames(shardNames, message.getStr("shards",null)); @@ -943,8 +941,6 @@ public class OverseerCollectionProcessor + " shards to be created (higher than the allowed number)"); } -// ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION, -// Overseer.CREATECOLLECTION, "name", message.getStr("name")); Overseer.getInQueue(zkStateReader.getZkClient()).offer(ZkStateReader.toJSON(message)); // wait for a while until we don't see the collection @@ -1102,11 +1098,4 @@ public class OverseerCollectionProcessor return isClosed; } - public static Map asMap(Object... vals) { - HashMap m = new HashMap(); - for(int i=0; i map = OverseerCollectionProcessor.asMap(QUEUE_OPERATION, CREATESHARD); + Map map = makeMap(QUEUE_OPERATION, CREATESHARD); copyIfNotNull(req.getParams(),map,COLLECTION_PROP, SHARD_ID_PROP, REPLICATION_FACTOR,CREATE_NODE_SET); ZkNodeProps m = new ZkNodeProps(map); handleResponse(CREATESHARD, m, rsp); } private static void copyIfNotNull(SolrParams params, Map props, String... keys) { + ArrayList prefixes = new ArrayList(1); if(keys !=null){ for (String key : keys) { + if(key.endsWith(".")) { + prefixes.add(key); + continue; + } String v = params.get(key); if(v != null) props.put(key,v); } } + if(prefixes.isEmpty()) return; + Iterator it = params.getParameterNamesIterator(); + String prefix = null; + for(;it.hasNext();){ + String name = it.next(); + for (int i = 0; i < prefixes.size(); i++) { + if(name.startsWith(prefixes.get(i))){ + String val = params.get(name); + if(val !=null) props.put(name,val); + } + } + } } Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/CustomCollectionTest.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/CustomCollectionTest.java?rev=1524446&r1=1524445&r2=1524446&view=diff ============================================================================== --- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/CustomCollectionTest.java (original) +++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/CustomCollectionTest.java Wed Sep 18 15:13:57 2013 @@ -183,8 +183,8 @@ public class CustomCollectionTest extend client = createCloudClient(COLL_PREFIX + i); } - Map props = OverseerCollectionProcessor.asMap( - ROUTER, ImplicitDocRouter.NAME, + Map props = ZkNodeProps.makeMap( + "router.name", ImplicitDocRouter.NAME, REPLICATION_FACTOR, replicationFactor, MAX_SHARDS_PER_NODE, maxShardsPerNode, SHARDS_PROP,"a,b,c"); @@ -216,7 +216,7 @@ public class CustomCollectionTest extend ClusterState clusterState = zkStateReader.getClusterState(); DocCollection coll = clusterState.getCollection(COLL_PREFIX + 0); - assertEquals("implicit", coll.getStr(ROUTER)); + assertEquals("implicit", ((Map)coll.get(ROUTER)).get("name") ); assertNotNull(coll.getStr(REPLICATION_FACTOR)); assertNotNull(coll.getStr(MAX_SHARDS_PER_NODE)); @@ -308,12 +308,12 @@ public class CustomCollectionTest extend String shard_fld = "shard_s"; try { client = createCloudClient(null); - Map props = OverseerCollectionProcessor.asMap( - ROUTER, ImplicitDocRouter.NAME, + Map props = ZkNodeProps.makeMap( + "router.name", ImplicitDocRouter.NAME, REPLICATION_FACTOR, replicationFactor, MAX_SHARDS_PER_NODE, maxShardsPerNode, SHARDS_PROP,"a,b,c,d", - DocRouter.ROUTE_FIELD, shard_fld); + "router.field", shard_fld); collectionName = COLL_PREFIX + "withShardField"; createCollection(collectionInfos, collectionName,props,client); @@ -371,11 +371,11 @@ public class CustomCollectionTest extend String shard_fld = "shard_s"; try { client = createCloudClient(null); - Map props = OverseerCollectionProcessor.asMap( + Map props = ZkNodeProps.makeMap( REPLICATION_FACTOR, replicationFactor, MAX_SHARDS_PER_NODE, maxShardsPerNode, NUM_SLICES,numShards, - DocRouter.ROUTE_FIELD, shard_fld); + "router.field", shard_fld); createCollection(collectionInfos, collectionName,props,client); } finally { Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/SliceStateUpdateTest.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/SliceStateUpdateTest.java?rev=1524446&r1=1524445&r2=1524446&view=diff ============================================================================== --- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/SliceStateUpdateTest.java (original) +++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/SliceStateUpdateTest.java Wed Sep 18 15:13:57 2013 @@ -151,7 +151,7 @@ public class SliceStateUpdateTest extend prop.put("state", "inactive"); Slice newSlice = new Slice(slice.getName(), slice.getReplicasMap(), prop); slicesMap.put(newSlice.getName(), newSlice); - props.put(DocCollection.DOC_ROUTER, ImplicitDocRouter.NAME); + props.put(DocCollection.DOC_ROUTER, ZkNodeProps.makeMap("name", ImplicitDocRouter.NAME)); DocCollection coll = new DocCollection("collection1", slicesMap, props, DocRouter.DEFAULT); collectionStates.put("collection1", coll); Modified: lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java?rev=1524446&r1=1524445&r2=1524446&view=diff ============================================================================== --- lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java (original) +++ lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java Wed Sep 18 15:13:57 2013 @@ -260,7 +260,8 @@ public class ClusterState implements JSO objs.remove(DocCollection.SHARDS); } - DocRouter router = DocRouter.getDocRouter(props.get(DocCollection.DOC_ROUTER)); + Map map = (Map) props.get(DocCollection.DOC_ROUTER); + DocRouter router = map == null ? DocRouter.DEFAULT : DocRouter.getDocRouter(map.get("name")); return new DocCollection(name, slices, props, router); } Modified: lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/CompositeIdRouter.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/CompositeIdRouter.java?rev=1524446&r1=1524445&r2=1524446&view=diff ============================================================================== --- lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/CompositeIdRouter.java (original) +++ lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/CompositeIdRouter.java Wed Sep 18 15:13:57 2013 @@ -61,7 +61,7 @@ public class CompositeIdRouter extends H @Override public int sliceHash(String id, SolrInputDocument doc, SolrParams params, DocCollection collection) { - String shardFieldName = collection ==null? null: collection.getStr(DocRouter.ROUTE_FIELD); + String shardFieldName = getRouteField(collection); String part1 = null; int idx = 0; int commaIdx = 0; Modified: lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/DocCollection.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/DocCollection.java?rev=1524446&r1=1524445&r2=1524446&view=diff ============================================================================== --- lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/DocCollection.java (original) +++ lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/DocCollection.java Wed Sep 18 15:13:57 2013 @@ -45,7 +45,7 @@ public class DocCollection extends ZkNod * @param props The properties of the slice. This is used directly and a copy is not made. */ public DocCollection(String name, Map slices, Map props, DocRouter router) { - super( props==null ? Collections.emptyMap() : props); + super( props==null ? props = new HashMap() : props); this.name = name; this.slices = slices; Modified: lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/DocRouter.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/DocRouter.java?rev=1524446&r1=1524445&r2=1524446&view=diff ============================================================================== --- lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/DocRouter.java (original) +++ lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/DocRouter.java Wed Sep 18 15:13:57 2013 @@ -28,9 +28,12 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import static org.apache.solr.common.cloud.DocCollection.DOC_ROUTER; + /** * Class to partition int range into n ranges. * @lucene.experimental @@ -38,12 +41,32 @@ import java.util.Map; public abstract class DocRouter { public static final String DEFAULT_NAME = CompositeIdRouter.NAME; public static final DocRouter DEFAULT = new CompositeIdRouter(); - public static final String ROUTE_FIELD = "routeField"; - public static DocRouter getDocRouter(Object routerSpec) { - DocRouter router = routerMap.get(routerSpec); + + public static DocRouter getDocRouter(Object routerName) { + DocRouter router = routerMap.get(routerName); if (router != null) return router; - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown document router '"+ routerSpec + "'"); + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown document router '"+ routerName + "'"); + } + + protected String getRouteField(DocCollection coll){ + if(coll == null) return null; + Map m = (Map) coll.get(DOC_ROUTER); + if(m == null) return null; + return (String) m.get("field"); + + } + + public static Map getRouterSpec(ZkNodeProps props){ + Map map = new LinkedHashMap(); + for (String s : props.keySet()) { + if(s.startsWith("router.")){ + map.put(s.substring(7), props.get(s)); + } + } + if(map.get("name") == null) map.put("name", DEFAULT_NAME); + return map; + } // currently just an implementation detail... Modified: lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ImplicitDocRouter.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ImplicitDocRouter.java?rev=1524446&r1=1524445&r2=1524446&view=diff ============================================================================== --- lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ImplicitDocRouter.java (original) +++ lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ImplicitDocRouter.java Wed Sep 18 15:13:57 2013 @@ -26,7 +26,6 @@ import org.slf4j.LoggerFactory; import java.util.Collection; import java.util.Collections; import java.util.List; - import static org.apache.solr.common.params.ShardParams._ROUTE_; /** This document router is for custom sharding @@ -42,7 +41,7 @@ public class ImplicitDocRouter extends D public Slice getTargetSlice(String id, SolrInputDocument sdoc, SolrParams params, DocCollection collection) { String shard = null; if (sdoc != null) { - String f = collection.getStr(ROUTE_FIELD); + String f = getRouteField(collection); if(f !=null) { Object o = sdoc.getFieldValue(f); if (o != null) shard = o.toString(); Modified: lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ZkNodeProps.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ZkNodeProps.java?rev=1524446&r1=1524445&r2=1524446&view=diff ============================================================================== --- lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ZkNodeProps.java (original) +++ lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ZkNodeProps.java Wed Sep 18 15:13:57 2013 @@ -58,7 +58,7 @@ public class ZkNodeProps implements JSON if ((keyVals.length & 0x01) != 0) { throw new IllegalArgumentException("arguments should be key,value"); } - Map propMap = new HashMap(keyVals.length>>1); + Map propMap = new LinkedHashMap(keyVals.length>>1); for (int i = 0; i < keyVals.length; i+=2) { propMap.put(keyVals[i].toString(), keyVals[i+1]); } @@ -125,7 +125,7 @@ public class ZkNodeProps implements JSON public Object get(String key) { return propMap.get(key); } - + @Override public String toString() { return JSONUtil.toJSON(this); Modified: lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java?rev=1524446&r1=1524445&r2=1524446&view=diff ============================================================================== --- lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java (original) +++ lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java Wed Sep 18 15:13:57 2013 @@ -1584,7 +1584,7 @@ public abstract class AbstractFullDistri String collectionName, int numShards, int numReplicas, int maxShardsPerNode, SolrServer client, String createNodeSetStr) throws SolrServerException, IOException { createCollection(collectionInfos, collectionName, - OverseerCollectionProcessor.asMap( + ZkNodeProps.makeMap( NUM_SLICES, numShards, REPLICATION_FACTOR, numReplicas, CREATE_NODE_SET, createNodeSetStr,