lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r1555629 [6/7] - in /lucene/dev/branches/lucene5376: ./ dev-tools/ dev-tools/maven/solr/contrib/map-reduce/ lucene/ lucene/analysis/ lucene/analysis/common/ lucene/analysis/common/src/java/org/apache/lucene/analysis/ckb/ lucene/analysis/com...
Date Sun, 05 Jan 2014 21:05:03 GMT
Modified: lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestFacets.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestFacets.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestFacets.java (original)
+++ lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestFacets.java Sun Jan  5 21:04:58 2014
@@ -110,8 +110,8 @@ public class TestFacets extends ServerBa
       put(o, "grouping", s);
     }
 
-    put(o, "facets", "[{path: 'dateFacet', topN: 10}]");
-    put(o, "retrieveFields", "['id', 'date', 'price', {field: 'body', highlight: " + (snippets ? "snippets" : "whole") + "}]");
+    put(o, "facets", "[{dim: dateFacet, topN: 10}]");
+    put(o, "retrieveFields", "[id, date, price, {field: body, highlight: " + (snippets ? "snippets" : "whole") + "}]");
 
     return send("search", o);
   }
@@ -188,19 +188,19 @@ public class TestFacets extends ServerBa
     long indexGen = getLong(send("addDocument", "{fields: {author: Tom}}"), "indexGen");
 
     // Initial query:
-    JSONObject o = send("search", String.format(Locale.ROOT, "{query: MatchAllDocsQuery, facets: [{path: [author], topN: 10}], searcher: {indexGen: %d}}", indexGen));
+    JSONObject o = send("search", String.format(Locale.ROOT, "{query: MatchAllDocsQuery, facets: [{dim: author, topN: 10}], searcher: {indexGen: %d}}", indexGen));
     assertEquals(6, o.get("totalHits"));
-    assertEquals("[[\"top\",0],[\"Tom\",3],[\"Lisa\",2],[\"Bob\",1]]", getArray(o, "facets[0].counts").toString());
+    assertEquals("[[\"top\",6],[\"Tom\",3],[\"Lisa\",2],[\"Bob\",1]]", getArray(o, "facets[0].counts").toString());
 
     // Now, single drill down:
-    o = send("search", String.format(Locale.ROOT, "{drillDowns: [{field: author, values: [Bob]}], query: MatchAllDocsQuery, facets: [{path: [author], topN: 10}], searcher: {indexGen: %d}}", indexGen));
+    o = send("search", String.format(Locale.ROOT, "{drillDowns: [{field: author, value: Bob}], query: MatchAllDocsQuery, facets: [{dim: author, topN: 10}], searcher: {indexGen: %d}}", indexGen));
     assertEquals(1, o.get("totalHits"));
-    assertEquals("[[\"top\",0],[\"Tom\",3],[\"Lisa\",2],[\"Bob\",1]]", getArray(o, "facets[0].counts").toString());
+    assertEquals("[[\"top\",6],[\"Tom\",3],[\"Lisa\",2],[\"Bob\",1]]", getArray(o, "facets[0].counts").toString());
 
-    // Multi drill down:
-    o = send("search", String.format(Locale.ROOT, "{drillDowns: [{field: author, values: [Bob, Lisa]}], query: MatchAllDocsQuery, facets: [{path: [author], topN: 10}], searcher: {indexGen: %d}}", indexGen));
+    // Multi (OR'd) drill down:
+    o = send("search", String.format(Locale.ROOT, "{drillDowns: [{field: author, value: Bob}, {field: author, value: Lisa}], query: MatchAllDocsQuery, facets: [{dim: author, topN: 10}], searcher: {indexGen: %d}}", indexGen));
     assertEquals(3, o.get("totalHits"));
-    assertEquals("[[\"top\",0],[\"Tom\",3],[\"Lisa\",2],[\"Bob\",1]]", getArray(o, "facets[0].counts").toString());
+    assertEquals("[[\"top\",6],[\"Tom\",3],[\"Lisa\",2],[\"Bob\",1]]", getArray(o, "facets[0].counts").toString());
   }
 
   public void testRangeFacets() throws Exception {
@@ -209,7 +209,7 @@ public class TestFacets extends ServerBa
     for(int i=0;i<100;i++) {
       gen = getLong(send("addDocument", "{fields: {longField: " + i + "}}"), "indexGen");
     }
-    JSONObject o = send("search", "{facets: [{path: longField, numericRanges: [{label: All, min: 0, max: 99, minInclusive: true, maxInclusive: true}, {label: Half, min: 0, max: 49, minInclusive: true, maxInclusive: true}]}], searcher: {indexGen: " + gen + "}}");
+    JSONObject o = send("search", "{facets: [{dim: longField, numericRanges: [{label: All, min: 0, max: 99, minInclusive: true, maxInclusive: true}, {label: Half, min: 0, max: 49, minInclusive: true, maxInclusive: true}]}], searcher: {indexGen: " + gen + "}}");
     assertEquals("All", getString(o, "facets[0].counts[1][0]"));
     assertEquals(100, getInt(o, "facets[0].counts[1][1]"));
     assertEquals("Half", getString(o, "facets[0].counts[2][0]"));

Modified: lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestGrouping.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestGrouping.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestGrouping.java (original)
+++ lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestGrouping.java Sun Jan  5 21:04:58 2014
@@ -199,7 +199,9 @@ public class TestGrouping extends Server
     if (sortField != null) {
       JSONObject sort = new JSONObject();
       o.put("sort", sort);
-      sort.put("doDocScores", true);
+      if (groupField == null) {
+        sort.put("doDocScores", true);
+      }
 
       JSONArray sortFields = new JSONArray();
       sort.put("fields", sortFields);
@@ -220,7 +222,7 @@ public class TestGrouping extends Server
       put(o, "grouping", s);
     }
 
-    put(o, "facets", "[{path: 'dateFacet', topN: 10}]");
+    put(o, "facets", "[{dim: 'dateFacet', topN: 10}]");
     put(o, "retrieveFields", "['id', 'date', 'price', {field: 'body', highlight: " + (snippets ? "snippets" : "whole") + "}]");
 
     return send("search", o);

Modified: lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestHighlight.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestHighlight.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestHighlight.java (original)
+++ lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestHighlight.java Sun Jan  5 21:04:58 2014
@@ -116,7 +116,7 @@ public class TestHighlight extends Serve
       put(o, "grouping", s);
     }
 
-    put(o, "facets", "[{path: 'dateFacet', topN: 10}]");
+    put(o, "facets", "[{dim: 'dateFacet', topN: 10}]");
     put(o, "retrieveFields", "['id', 'date', 'price', {field: 'body', highlight: " + (snippets ? "snippets" : "whole") + "}]");
 
     return send("search", o);

Modified: lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestIndexing.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestIndexing.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestIndexing.java (original)
+++ lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestIndexing.java Sun Jan  5 21:04:58 2014
@@ -132,11 +132,7 @@ public class TestIndexing extends Server
         sb.append(',');
       }
       if (i == 57) {
-        JSONArray broken = new JSONArray();
-        broken.add("2013");
-        broken.add("");
-        broken.add("17");
-        o.put("dateFacet", broken);
+        o.put("foobar", 17);
       }
       JSONObject o2 = new JSONObject();
       o2.put("fields", o);
@@ -188,8 +184,8 @@ public class TestIndexing extends Server
       put(o, "grouping", s);
     }
 
-    put(o, "facets", "[{path: 'dateFacet', topN: 10}]");
-    put(o, "retrieveFields", "['id', 'date', 'price', {field: 'body', highlight: " + (snippets ? "snippets" : "whole") + "}]");
+    put(o, "facets", "[{dim: dateFacet, topN: 10}]");
+    put(o, "retrieveFields", "[id, date, price, {field: body, highlight: " + (snippets ? "snippets" : "whole") + "}]");
 
     return send("search", o);
   }

Modified: lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestNumericFields.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestNumericFields.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestNumericFields.java (original)
+++ lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestNumericFields.java Sun Jan  5 21:04:58 2014
@@ -25,6 +25,7 @@ public class TestNumericFields extends S
 
   @BeforeClass
   public static void initClass() throws Exception {
+    curIndexName = "index";
     startServer();
     createAndStartIndex();
     registerFields();

Modified: lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestSearch.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestSearch.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestSearch.java (original)
+++ lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestSearch.java Sun Jan  5 21:04:58 2014
@@ -167,6 +167,7 @@ public class TestSearch extends ServerBa
   public void testNumericRangeQuery() throws Exception {
     curIndexName = "nrq";
     for(String type : new String[] {"int", "long", "float", "double"}) {
+      _TestUtil.rmDir(new File("nrq"));
       send("createIndex", "{rootDir: nrq}");
       send("startIndex", "{}");
       send("registerFields", String.format(Locale.ROOT, "{fields: {nf: {type: %s, index: true}}}", type));

Modified: lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestServer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestServer.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestServer.java (original)
+++ lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestServer.java Sun Jan  5 21:04:58 2014
@@ -29,6 +29,7 @@ public class TestServer extends ServerBa
 
   @BeforeClass
   public static void initClass() throws Exception {
+    curIndexName = "index";
     startServer();
     createAndStartIndex();
     registerFields();
@@ -118,8 +119,8 @@ public class TestServer extends ServerBa
       put(o, "grouping", s);
     }
 
-    put(o, "facets", "[{path: 'dateFacet', topN: 10}]");
-    put(o, "retrieveFields", "['id', 'date', 'price', {field: 'body', highlight: " + (snippets ? "snippets" : "whole") + "}]");
+    put(o, "facets", "[{dim: dateFacet, topN: 10}]");
+    put(o, "retrieveFields", "[id, date, price, {field: body, highlight: " + (snippets ? "snippets" : "whole") + "}]");
 
     return send("search", o);
   }

Modified: lucene/dev/branches/lucene5376/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/CHANGES.txt?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/CHANGES.txt (original)
+++ lucene/dev/branches/lucene5376/solr/CHANGES.txt Sun Jan  5 21:04:58 2014
@@ -83,7 +83,7 @@ Detailed Change List
 New Features
 ----------------------
 
-* SOLR-5308: A new 'migrate' collection API to split all documents with a
+* SOLR-5308: SOLR-5601: A new 'migrate' collection API to split all documents with a
   route key into another collection (shalin)
 
 * SOLR-5441: Expose number of transaction log files and their size via JMX.
@@ -132,18 +132,16 @@ New Features
 * SOLR-5556: Allow class of CollectionsHandler and InfoHandler to be specified
   in solr.xml. (Gregory Chanan, Alan Woodward)
 
+* SOLR-5581: Give ZkCLI the ability to get files. (Gregory Chanan via Mark Miller)
+
+* SOLR-5536: Add ValueSource collapse criteria to CollapsingQParsingPlugin (Joel Bernstein)
+
 Bug Fixes
 ----------------------
 
 * SOLR-5438: DebugComponent throws NPE when used with grouping.
   (Tomás Fernández Löbbe via shalin)
 
-* SOLR-5442: Python client cannot parse proxied response when served by Tomcat.
-  (Patrick Hunt, Gregory Chanan, Vamsee Yarlagadda, Romain Rigaux, Mark Miller)
-
-* SOLR-5445: Proxied responses should propagate all headers rather than the 
-  first one for each key. (Patrick Hunt, Mark Miller) 
-
 * SOLR-4612: Admin UI - Analysis Screen contains empty table-columns (steffkes)
 
 * SOLR-5451: SyncStrategy closes it's http connection manager before the
@@ -155,15 +153,9 @@ Bug Fixes
 * SOLR-5461: Request proxying should only set con.setDoOutput(true) if the
   request is a post. (Mark Miller)
 
-* SOLR-5479: SolrCmdDistributor retry logic stops if a leader for the request 
-  cannot be found in 1 second. (Mark Miller)
-
 * SOLR-5481: SolrCmdDistributor should not let the http client do it's own 
   retries. (Mark Miller)
 
-* SOLR-4709: The core reload after replication if config files have changed
-  can fail due to a race condition. (Mark Miller, Hossman))
-
 * LUCENE-5347: Fixed Solr's Zookeeper Client to copy files to Zookeeper using
   binary transfer. Previously data was read with default encoding and stored
   in zookeeper as UTF-8. This bug was found after upgrading to forbidden-apis
@@ -173,13 +165,6 @@ Bug Fixes
   a delta-import is run first before any full-imports.
   (Sebastien Lorber, Arcadius Ahouansou via shalin)
 
-* SOLR-5496: We should share an http connection manager across non search 
-  HttpClients and ensure all http connection managers get shutdown.
-  (Mark Miller)
-
-* SOLR-5503: Retry 'forward to leader' requests less aggressively - rather 
-  than on IOException and status 500, ConnectException. (Mark Miller)
-
 * SOLR-5494: CoreContainer#remove throws NPE rather than returning null when
   a SolrCore does not exist in core discovery mode. (Mark Miller)
   
@@ -200,20 +185,19 @@ Bug Fixes
 * SOLR-5540: HdfsLockFactory should explicitly create the lock parent directory 
   if necessary. (Mark Miller)
 
-* SOLR-5532: SolrJ Content-Type validation is too strict for some
-  webcontainers / proxies. (Jakob Furrer, hossman, Shawn Heisey, Uwe Schindler,
-  Mark Miller)
-
-* SOLR-5547: Creating a collection alias using SolrJ's CollectionAdminRequest
-  sets the alias name and the collections to alias to the same value.
-  (Aaron Schram, Mark Miller)
-
 * SOLR-5543: Core swaps resulted in duplicate core entries in solr.xml when 
   using solr.xml persistence. (Bill Bell, Alan Woodward)
 
 * SOLR-5562: ConcurrentUpdateSolrServer constructor ignores supplied httpclient.
   (Kyle Halliday via Mark Miller)
-  
+
+* SOLR-5567: ZkController getHostAddress duplicates url prefix.
+  (Kyle Halliday, Alexey Serba, shalin)
+
+* SOLR-5577: Likely ZooKeeper expiration should not slow down updates a given
+  amount, but instead cut off updates after a given time. 
+  (Mark Miller, Christine Poerschke)
+
 Optimizations
 ----------------------
 
@@ -227,8 +211,11 @@ Optimizations
 
 * SOLR-5512: Optimize DocValuesFacets. (Robert Muir)
 
-* SOLR-5576: Improve concurrency when registering and waiting for all 
-  SolrCore's to register a DOWN state. (Christine Poerschke via Mark Miller)
+* SOLR-2960: fix DIH XPathEntityProcessor to add the correct number of "null"
+  placeholders for multi-valued fields (Michael Watts via James Dyer)
+
+* SOLR-5214: Reduce memory usage for shard splitting by merging segments one
+  at a time. (Christine Poerschke via shalin)
 
 Other Changes
 ---------------------
@@ -271,6 +258,12 @@ Other Changes
 * SOLR-5574: CoreContainer shutdown publishes all nodes as down and waits to
   see that and then again publishes all nodes as down. (Mark Miller)
 
+* SOLR-5590: Upgrade HttpClient/HttpComponents to 4.3.x.
+  (Karl Wright via Shawn Heisey)
+
+* pull request #11: change the default of hl.phraseLimit to 5000.
+  (Michael Della Bitta via Robert Muir)
+
 ==================  4.6.1  ==================
 
 Versions of Major Components
@@ -292,7 +285,68 @@ Bug Fixes
 
 * SOLR-5416: CollapsingQParserPlugin breaks Tag/Exclude Faceting (David Boychuck, Joel Bernstein)
 
+* SOLR-5442: Python client cannot parse proxied response when served by Tomcat.
+  (Patrick Hunt, Gregory Chanan, Vamsee Yarlagadda, Romain Rigaux, Mark Miller)
+
+* SOLR-5445: Proxied responses should propagate all headers rather than the 
+  first one for each key. (Patrick Hunt, Mark Miller) 
+
+* SOLR-5479: SolrCmdDistributor retry logic stops if a leader for the request 
+  cannot be found in 1 second. (Mark Miller)
+
+* SOLR-5532: SolrJ Content-Type validation is too strict for some
+  webcontainers / proxies. (Jakob Furrer, hossman, Shawn Heisey, Uwe Schindler,
+  Mark Miller)
+
+* SOLR-5547: Creating a collection alias using SolrJ's CollectionAdminRequest
+  sets the alias name and the collections to alias to the same value.
+  (Aaron Schram, Mark Miller)
+
+* SOLR-5580: NPE when creating a core with both explicit shard and coreNodeName.
+  (YouPeng Yang, Mark Miller)
+
+* SOLR-5552: Leader recovery process can select the wrong leader if all replicas 
+  for a shard are down and trying to recover as well as lose updates that should 
+  have been recovered. (Timothy Potter, Mark Miller)
+  
+* SOLR-5569 A replica should not try and recover from a leader until it has 
+  published that it is ACTIVE. (Mark Miller)
+  
+* SOLR-5568 A SolrCore cannot decide to be the leader just because the cluster
+  state says no other SolrCore's are active. (Mark Miller)
+
+* SOLR-5496: We should share an http connection manager across non search 
+  HttpClients and ensure all http connection managers get shutdown.
+  (Mark Miller)
+
+* SOLR-5583: ConcurrentUpdateSolrServer#blockUntilFinished may wait forever if
+  the executor service is shutdown. (Mark Miller)
+
+* SOLR-5586: All ZkCmdExecutor's should be initialized with the zk client
+  timeout. (Mark Miller)
+
+* SOLR-5587: ElectionContext implementations should use 
+  ZkCmdExecutor#ensureExists to ensure their election paths are properly
+  created. (Mark Miller)
+
+* SOLR-4709: The core reload after replication if config files have changed
+  can fail due to a race condition. (Mark Miller, Hossman))
+
+* SOLR-5503: Retry 'forward to leader' requests less aggressively - rather 
+  than on IOException and status 500, ConnectException. (Mark Miller)
+
+* SOLR-5588: PeerSync doesn't count all connect failures as success.
+  (Mark Miller)
+    
+* SOLR-5564: hl.maxAlternateFieldLength should apply to original field when
+  fallback is attempted (janhoy)
+
+Optimizations
+----------------------
 
+* SOLR-5576: Improve concurrency when registering and waiting for all 
+  SolrCore's to register a DOWN state. (Christine Poerschke via Mark Miller)
+  
 ==================  4.6.0 ==================
 
 Versions of Major Components

Modified: lucene/dev/branches/lucene5376/solr/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/build.xml?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/build.xml (original)
+++ lucene/dev/branches/lucene5376/solr/build.xml Sun Jan  5 21:04:58 2014
@@ -612,7 +612,7 @@
   
   <property name="analysis-common.res.dir"  value="../lucene/analysis/common/src/resources/org/apache/lucene/analysis"/>
   <property name="analysis-kuromoji.res.dir"  value="../lucene/analysis/kuromoji/src/resources/org/apache/lucene/analysis"/>
-  <property name="analysis.conf.dest" value="${example}/solr/conf/lang"/>
+  <property name="analysis.conf.dest" value="${example}/solr/collection1/conf/lang"/>
 
   <target name="sync-analyzers"
           description="Committers' Helper: synchronizes analysis resources (e.g. stoplists) to the example">
@@ -625,6 +625,9 @@
     <!-- catalan -->
     <copy verbose="true" file="${analysis-common.res.dir}/ca/stopwords.txt"
                          tofile="${analysis.conf.dest}/stopwords_ca.txt"/>
+    <!-- kurdish -->
+    <copy verbose="true" file="${analysis-common.res.dir}/ckb/stopwords.txt"
+                         tofile="${analysis.conf.dest}/stopwords_ckb.txt"/>
     <!-- czech -->
     <copy verbose="true" file="${analysis-common.res.dir}/cz/stopwords.txt"
                          tofile="${analysis.conf.dest}/stopwords_cz.txt"/>

Modified: lucene/dev/branches/lucene5376/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/XPathRecordReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/XPathRecordReader.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/XPathRecordReader.java (original)
+++ lucene/dev/branches/lucene5376/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/XPathRecordReader.java Sun Jan  5 21:04:58 2014
@@ -296,7 +296,7 @@ public class XPathRecordReader {
                 for (Node n : childNodes) {
                   // For the multivalue child nodes where we could have, but
                   // didnt, collect text. Push a null string into values.
-                  if (!childrenFound.contains(n)) n.putNulls(values);
+                  if (!childrenFound.contains(n)) n.putNulls(values, valuesAddedinThisFrame);
                 }
               }
               return;
@@ -429,18 +429,28 @@ public class XPathRecordReader {
      * pushing a null string onto every multiValued fieldName's List of values
      * where a value has not been provided from the stream.
      */
-    private void putNulls(Map<String, Object> values) {
+    private void putNulls(Map<String, Object> values, Set<String> valuesAddedinThisFrame) {
       if (attributes != null) {
         for (Node n : attributes) {
-          if (n.multiValued)
-            putText(values, null, n.fieldName, true);
+          if (n.multiValued) {
+            putANull(n.fieldName, values, valuesAddedinThisFrame);
+          }
         }
       }
-      if (hasText && multiValued)
-        putText(values, null, fieldName, true);
+      if (hasText && multiValued) {
+        putANull(fieldName, values, valuesAddedinThisFrame);
+      }
       if (childNodes != null) {
-        for (Node childNode : childNodes)
-          childNode.putNulls(values);
+        for (Node childNode : childNodes) {
+          childNode.putNulls(values, valuesAddedinThisFrame);
+        }
+      }
+    }
+    
+    private void putANull(String thisFieldName, Map<String, Object> values, Set<String> valuesAddedinThisFrame) {
+      putText(values, null, thisFieldName, true);
+      if( valuesAddedinThisFrame != null) {
+        valuesAddedinThisFrame.add(thisFieldName);
       }
     }
 

Modified: lucene/dev/branches/lucene5376/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestXPathEntityProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestXPathEntityProcessor.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestXPathEntityProcessor.java (original)
+++ lucene/dev/branches/lucene5376/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestXPathEntityProcessor.java Sun Jan  5 21:04:58 2014
@@ -93,6 +93,128 @@ public class TestXPathEntityProcessor ex
     assertEquals("2", l.get(1));
     assertEquals("ü", l.get(2));
   }
+  
+  @SuppressWarnings({"rawtypes", "unchecked"})
+  @Test
+  public void testMultiValuedWithMultipleDocuments() throws Exception {
+    Map entityAttrs = createMap("name", "e", "url", "testdata.xml", XPathEntityProcessor.FOR_EACH, "/documents/doc");
+    List fields = new ArrayList();
+    fields.add(createMap("column", "id", "xpath", "/documents/doc/id", DataImporter.MULTI_VALUED, "false"));
+    fields.add(createMap("column", "a", "xpath", "/documents/doc/a", DataImporter.MULTI_VALUED, "true"));
+    fields.add(createMap("column", "s1dataA", "xpath", "/documents/doc/sec1/s1dataA", DataImporter.MULTI_VALUED, "true"));
+    fields.add(createMap("column", "s1dataB", "xpath", "/documents/doc/sec1/s1dataB", DataImporter.MULTI_VALUED, "true")); 
+    fields.add(createMap("column", "s1dataC", "xpath", "/documents/doc/sec1/s1dataC", DataImporter.MULTI_VALUED, "true")); 
+    
+    Context c = getContext(null,
+            new VariableResolver(), getDataSource(textMultipleDocuments), Context.FULL_DUMP, fields, entityAttrs);
+    XPathEntityProcessor xPathEntityProcessor = new XPathEntityProcessor();
+    xPathEntityProcessor.init(c);
+    List<Map<String, Object>> result = new ArrayList<Map<String, Object>>();
+    while (true) {
+      Map<String, Object> row = xPathEntityProcessor.nextRow();
+      if (row == null)
+        break;
+      result.add(row);
+    }
+    {  
+      assertEquals("1", result.get(0).get("id"));
+      List a = (List)result.get(0).get("a");
+      List s1dataA = (List)result.get(0).get("s1dataA");
+      List s1dataB = (List)result.get(0).get("s1dataB");
+      List s1dataC = (List)result.get(0).get("s1dataC");      
+      assertEquals(2, a.size());
+      assertEquals("id1-a1", a.get(0));
+      assertEquals("id1-a2", a.get(1));
+      assertEquals(3, s1dataA.size());
+      assertEquals("id1-s1dataA-1", s1dataA.get(0));
+      assertNull(s1dataA.get(1));
+      assertEquals("id1-s1dataA-3", s1dataA.get(2));
+      assertEquals(3, s1dataB.size());
+      assertEquals("id1-s1dataB-1", s1dataB.get(0));
+      assertEquals("id1-s1dataB-2", s1dataB.get(1));
+      assertEquals("id1-s1dataB-3", s1dataB.get(2));
+      assertEquals(3, s1dataC.size());
+      assertNull(s1dataC.get(0));
+      assertNull(s1dataC.get(1));
+      assertNull(s1dataC.get(2));
+    }
+    { 
+      assertEquals("2", result.get(1).get("id"));
+      List a = (List)result.get(1).get("a");
+      List s1dataA = (List)result.get(1).get("s1dataA");
+      List s1dataB = (List)result.get(1).get("s1dataB");
+      List s1dataC = (List)result.get(1).get("s1dataC");  
+      assertTrue(a==null || a.size()==0);
+      assertEquals(1, s1dataA.size()); 
+      assertNull(s1dataA.get(0));
+      assertEquals(1, s1dataB.size());
+      assertEquals("id2-s1dataB-1", s1dataB.get(0));
+      assertEquals(1, s1dataC.size());
+      assertNull(s1dataC.get(0));
+    }  
+    {
+      assertEquals("3", result.get(2).get("id"));
+      List a = (List)result.get(2).get("a");
+      List s1dataA = (List)result.get(2).get("s1dataA");
+      List s1dataB = (List)result.get(2).get("s1dataB");
+      List s1dataC = (List)result.get(2).get("s1dataC");  
+      assertTrue(a==null || a.size()==0);
+      assertEquals(1, s1dataA.size());
+      assertEquals("id3-s1dataA-1", s1dataA.get(0));
+      assertEquals(1, s1dataB.size());
+      assertNull(s1dataB.get(0));
+      assertEquals(1, s1dataC.size());
+      assertNull(s1dataC.get(0)); 
+    }
+    {  
+      assertEquals("4", result.get(3).get("id"));
+      List a = (List)result.get(3).get("a");
+      List s1dataA = (List)result.get(3).get("s1dataA");
+      List s1dataB = (List)result.get(3).get("s1dataB");
+      List s1dataC = (List)result.get(3).get("s1dataC");  
+      assertTrue(a==null || a.size()==0);
+      assertEquals(1, s1dataA.size());
+      assertEquals("id4-s1dataA-1", s1dataA.get(0));
+      assertEquals(1, s1dataB.size());
+      assertEquals("id4-s1dataB-1", s1dataB.get(0));
+      assertEquals(1, s1dataC.size());
+      assertEquals("id4-s1dataC-1", s1dataC.get(0));
+    }
+    {
+      assertEquals("5", result.get(4).get("id"));
+      List a = (List)result.get(4).get("a");
+      List s1dataA = (List)result.get(4).get("s1dataA");
+      List s1dataB = (List)result.get(4).get("s1dataB");
+      List s1dataC = (List)result.get(4).get("s1dataC");  
+      assertTrue(a==null || a.size()==0);      
+      assertEquals(1, s1dataA.size());
+      assertNull(s1dataA.get(0)); 
+      assertEquals(1, s1dataB.size());
+      assertNull(s1dataB.get(0)); 
+      assertEquals(1, s1dataC.size());
+      assertEquals("id5-s1dataC-1", s1dataC.get(0));
+    }
+    {  
+      assertEquals("6", result.get(5).get("id"));
+      List a = (List)result.get(5).get("a");
+      List s1dataA = (List)result.get(5).get("s1dataA");
+      List s1dataB = (List)result.get(5).get("s1dataB");
+      List s1dataC = (List)result.get(5).get("s1dataC");     
+      assertTrue(a==null || a.size()==0); 
+      assertEquals(3, s1dataA.size());
+      assertEquals("id6-s1dataA-1", s1dataA.get(0));
+      assertEquals("id6-s1dataA-2", s1dataA.get(1));
+      assertNull(s1dataA.get(2));
+      assertEquals(3, s1dataB.size());
+      assertEquals("id6-s1dataB-1", s1dataB.get(0));
+      assertEquals("id6-s1dataB-2", s1dataB.get(1));
+      assertEquals("id6-s1dataB-3", s1dataB.get(2));
+      assertEquals(3, s1dataC.size());
+      assertEquals("id6-s1dataC-1", s1dataC.get(0));
+      assertNull(s1dataC.get(1));
+      assertEquals("id6-s1dataC-3", s1dataC.get(2));
+    }
+  }
 
   @Test
   public void testMultiValuedFlatten() throws Exception  {
@@ -305,4 +427,68 @@ public class TestXPathEntityProcessor ex
   private static final String testXml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!DOCTYPE root [\n<!ENTITY uuml \"&#252;\" >\n]>\n<root><a>1</a><a>2</a><a>&uuml;</a></root>";
 
   private static final String testXmlFlatten = "<?xml version=\"1.0\" encoding=\"UTF-8\"?><root><a>1<b>B</b>2</a></root>";
+  
+  private static final String textMultipleDocuments = 
+      "<?xml version=\"1.0\" ?>" +
+          "<documents>" +          
+          " <doc>" +
+          "  <id>1</id>" +
+          "  <a>id1-a1</a>" +
+          "  <a>id1-a2</a>" +
+          "  <sec1>" +
+          "   <s1dataA>id1-s1dataA-1</s1dataA>" +
+          "   <s1dataB>id1-s1dataB-1</s1dataB>" +
+          "  </sec1>" +
+          "  <sec1>" +
+          "   <s1dataB>id1-s1dataB-2</s1dataB>" +
+          "  </sec1>" +
+          "  <sec1>" +
+          "   <s1dataA>id1-s1dataA-3</s1dataA>" +
+          "   <s1dataB>id1-s1dataB-3</s1dataB>" +
+          "  </sec1>" +
+          " </doc>" +
+          " <doc>" +
+          "  <id>2</id>" +          
+          "  <sec1>" +
+          "   <s1dataB>id2-s1dataB-1</s1dataB>" +
+          "  </sec1>" + 
+          " </doc>" +
+          " <doc>" +
+          "  <id>3</id>" +          
+          "  <sec1>" +
+          "   <s1dataA>id3-s1dataA-1</s1dataA>" +
+          "  </sec1>" + 
+          " </doc>" +
+          " <doc>" +
+          "  <id>4</id>" +          
+          "  <sec1>" +
+          "   <s1dataA>id4-s1dataA-1</s1dataA>" +
+          "   <s1dataB>id4-s1dataB-1</s1dataB>" +
+          "   <s1dataC>id4-s1dataC-1</s1dataC>" +
+          "  </sec1>" + 
+          " </doc>" +
+          " <doc>" +
+          "  <id>5</id>" +          
+          "  <sec1>" +
+          "   <s1dataC>id5-s1dataC-1</s1dataC>" +
+          "  </sec1>" + 
+          " </doc>" +
+          " <doc>" +
+          "  <id>6</id>" +
+          "  <sec1>" +
+          "   <s1dataA>id6-s1dataA-1</s1dataA>" +
+          "   <s1dataB>id6-s1dataB-1</s1dataB>" +
+          "   <s1dataC>id6-s1dataC-1</s1dataC>" +
+          "  </sec1>" +
+          "  <sec1>" +
+          "   <s1dataA>id6-s1dataA-2</s1dataA>" +
+          "   <s1dataB>id6-s1dataB-2</s1dataB>" +
+          "  </sec1>" +
+          "  <sec1>" +
+          "   <s1dataB>id6-s1dataB-3</s1dataB>" +
+          "   <s1dataC>id6-s1dataC-3</s1dataC>" +
+          "  </sec1>" +
+          " </doc>" +
+          "</documents>"
+         ;
 }

Modified: lucene/dev/branches/lucene5376/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestZKPropertiesWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestZKPropertiesWriter.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestZKPropertiesWriter.java (original)
+++ lucene/dev/branches/lucene5376/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestZKPropertiesWriter.java Sun Jan  5 21:04:58 2014
@@ -17,9 +17,19 @@ package org.apache.solr.handler.dataimpo
  * limitations under the License.
  */
 
+import java.io.File;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
 import org.apache.solr.cloud.AbstractZkTestCase;
 import org.apache.solr.cloud.ZkTestServer;
 import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.core.CoreContainer;
 import org.apache.solr.request.LocalSolrQueryRequest;
 import org.apache.solr.request.SolrQueryRequest;
 import org.junit.After;
@@ -29,20 +39,13 @@ import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
-import java.io.File;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-
 public class TestZKPropertiesWriter extends AbstractDataImportHandlerTestCase {
   protected static ZkTestServer zkServer;
 
   protected static String zkDir;
 
+  private static CoreContainer cc;
+
   private String dateFormat = "yyyy-MM-dd HH:mm:ss.SSSSSS";
 
   @BeforeClass
@@ -61,13 +64,12 @@ public class TestZKPropertiesWriter exte
         "dataimport-solrconfig.xml", "dataimport-schema.xml");
 
     //initCore("solrconfig.xml", "schema.xml", getFile("dih/solr").getAbsolutePath());
-    createDefaultCoreContainer(getFile("dih/solr").getAbsolutePath());
+    cc = createDefaultCoreContainer(getFile("dih/solr").getAbsolutePath());
   }
 
   @Before
   public void beforeDihZKTest() throws Exception {
-    clearIndex();
-    assertU(commit());
+
   }
 
   @After
@@ -78,13 +80,12 @@ public class TestZKPropertiesWriter exte
 
   @AfterClass
   public static void dihZk_afterClass() throws Exception {
+    cc.shutdown();
+    
     zkServer.shutdown();
 
     zkServer = null;
     zkDir = null;
-
-    // wait just a bit for any zk client threads to outlast timeout
-    Thread.sleep(2000);
   }
 
   @Test

Modified: lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/DistributedQueue.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/DistributedQueue.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/DistributedQueue.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/DistributedQueue.java Sun Jan  5 21:04:58 2014
@@ -56,7 +56,7 @@ public class DistributedQueue {
   public DistributedQueue(SolrZkClient zookeeper, String dir, List<ACL> acl) {
     this.dir = dir;
     
-    ZkCmdExecutor cmdExecutor = new ZkCmdExecutor(30);
+    ZkCmdExecutor cmdExecutor = new ZkCmdExecutor(zookeeper.getZkClientTimeout());
     try {
       cmdExecutor.ensureExists(dir, zookeeper);
     } catch (KeeperException e) {

Modified: lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java Sun Jan  5 21:04:58 2014
@@ -10,6 +10,7 @@ import org.apache.solr.common.cloud.Clus
 import org.apache.solr.common.cloud.Replica;
 import org.apache.solr.common.cloud.Slice;
 import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.solr.common.cloud.ZkCmdExecutor;
 import org.apache.solr.common.cloud.ZkCoreNodeProps;
 import org.apache.solr.common.cloud.ZkNodeProps;
 import org.apache.solr.common.cloud.ZkStateReader;
@@ -93,6 +94,15 @@ class ShardLeaderElectionContextBase ext
     this.zkClient = zkStateReader.getZkClient();
     this.shardId = shardId;
     this.collection = collection;
+    
+    try {
+      new ZkCmdExecutor(zkStateReader.getZkClient().getZkClientTimeout()).ensureExists(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collection, zkClient);
+    } catch (KeeperException e) {
+      throw new SolrException(ErrorCode.SERVER_ERROR, e);
+    } catch (InterruptedException e) {
+      Thread.currentThread().interrupt();
+      throw new SolrException(ErrorCode.SERVER_ERROR, e);
+    }
   }
 
   @Override
@@ -424,6 +434,14 @@ final class OverseerElectionContext exte
     super(zkNodeName, "/overseer_elect", "/overseer_elect/leader", null, zkClient);
     this.overseer = overseer;
     this.zkClient = zkClient;
+    try {
+      new ZkCmdExecutor(zkClient.getZkClientTimeout()).ensureExists("/overseer_elect", zkClient);
+    } catch (KeeperException e) {
+      throw new SolrException(ErrorCode.SERVER_ERROR, e);
+    } catch (InterruptedException e) {
+      Thread.currentThread().interrupt();
+      throw new SolrException(ErrorCode.SERVER_ERROR, e);
+    }
   }
 
   @Override

Modified: lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java Sun Jan  5 21:04:58 2014
@@ -69,7 +69,7 @@ public  class LeaderElector {
 
   public LeaderElector(SolrZkClient zkClient) {
     this.zkClient = zkClient;
-    zkCmdExecutor = new ZkCmdExecutor((int) (zkClient.getZkClientTimeout()/1000.0 + 3000));
+    zkCmdExecutor = new ZkCmdExecutor(zkClient.getZkClientTimeout());
   }
   
   // for tests

Modified: lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/Overseer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/Overseer.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/Overseer.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/Overseer.java Sun Jan  5 21:04:58 2014
@@ -442,18 +442,10 @@ public class Overseer {
             // if coreNodeName is null, auto assign one
             coreNodeName = Assign.assignNode(collection, state);
           }
-          message.getProperties().put(ZkStateReader.CORE_NODE_NAME_PROP, coreNodeName);
-        } else {
-          //probably, this core was removed explicitly
-          if (sliceName !=null && collectionExists &&  !"true".equals(state.getCollection(collection).getStr("autoCreated"))) {
-            Slice slice = state.getSlice(collection, sliceName);
-            if (slice.getReplica(coreNodeName) == null) {
-              log.info("core_deleted . Just return");
-              return state;
-            }
-          }
-
+          message.getProperties().put(ZkStateReader.CORE_NODE_NAME_PROP,
+              coreNodeName);
         }
+
         // use the provided non null shardId
         if (sliceName == null) {
           //get shardId from ClusterState

Modified: lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java Sun Jan  5 21:04:58 2014
@@ -1191,7 +1191,7 @@ public class OverseerCollectionProcessor
     // refresh cluster state
     clusterState = zkStateReader.getClusterState();
     Slice tempSourceSlice = clusterState.getCollection(tempSourceCollectionName).getSlices().iterator().next();
-    Replica tempSourceLeader = clusterState.getLeader(tempSourceCollectionName, tempSourceSlice.getName());
+    Replica tempSourceLeader = zkStateReader.getLeaderRetry(tempSourceCollectionName, tempSourceSlice.getName(), 60000);
 
     String tempCollectionReplica1 = tempSourceCollectionName + "_" + tempSourceSlice.getName() + "_replica1";
     String coreNodeName = waitForCoreNodeName(clusterState.getCollection(tempSourceCollectionName),

Modified: lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java Sun Jan  5 21:04:58 2014
@@ -3,6 +3,7 @@ package org.apache.solr.cloud;
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.CommandLineParser;
 import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.cli.Option;
 import org.apache.commons.cli.OptionBuilder;
@@ -48,6 +49,8 @@ public class ZkCLI {
   private static final String MAKEPATH = "makepath";
   private static final String PUT = "put";
   private static final String PUT_FILE = "putfile";
+  private static final String GET = "get";
+  private static final String GET_FILE = "getfile";
   private static final String DOWNCONFIG = "downconfig";
   private static final String ZK_CLI_NAME = "ZkCLI";
   private static final String HELP = "help";
@@ -92,7 +95,7 @@ public class ZkCLI {
         .withDescription(
             "cmd to run: " + BOOTSTRAP + ", " + UPCONFIG + ", " + DOWNCONFIG
                 + ", " + LINKCONFIG + ", " + MAKEPATH + ", " + PUT + ", " + PUT_FILE + ","
-                + LIST + ", " + CLEAR).create(CMD));
+                + GET + "," + GET_FILE + ", " + LIST + ", " + CLEAR).create(CMD));
 
     Option zkHostOption = new Option("z", ZKHOST, true,
         "ZooKeeper host address");
@@ -137,6 +140,8 @@ public class ZkCLI {
         System.out.println("zkcli.sh -zkhost localhost:9983 -cmd " + MAKEPATH + " /apache/solr");
         System.out.println("zkcli.sh -zkhost localhost:9983 -cmd " + PUT + " /solr.conf 'conf data'");
         System.out.println("zkcli.sh -zkhost localhost:9983 -cmd " + PUT_FILE + " /solr.xml /User/myuser/solr/solr.xml");
+        System.out.println("zkcli.sh -zkhost localhost:9983 -cmd " + GET + " /solr.xml");
+        System.out.println("zkcli.sh -zkhost localhost:9983 -cmd " + GET_FILE + " /solr.xml solr.xml.file");
         System.out.println("zkcli.sh -zkhost localhost:9983 -cmd " + CLEAR + " /solr");
         System.out.println("zkcli.sh -zkhost localhost:9983 -cmd " + LIST);
         return;
@@ -264,6 +269,22 @@ public class ZkCLI {
             IOUtils.closeQuietly(is);
           }
 
+        } else if (line.getOptionValue(CMD).equals(GET)) {
+          List arglist = line.getArgList();
+          if (arglist.size() != 1) {
+            System.out.println("-" + GET + " requires one arg - the path to get");
+            System.exit(1);
+          }
+          byte [] data = zkClient.getData(arglist.get(0).toString(), null, null, true);
+          System.out.println(new String(data, "UTF-8"));
+        } else if (line.getOptionValue(CMD).equals(GET_FILE)) {
+          List arglist = line.getArgList();
+          if (arglist.size() != 2) {
+            System.out.println("-" + GET_FILE + "requires two args - the path to get and the file to save it to");
+            System.exit(1);
+          }
+          byte [] data = zkClient.getData(arglist.get(0).toString(), null, null, true);
+          FileUtils.writeByteArrayToFile(new File(arglist.get(1).toString()), data);
         }
       } finally {
         if (solrPort != null) {

Modified: lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/ZkController.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/ZkController.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/ZkController.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/ZkController.java Sun Jan  5 21:04:58 2014
@@ -477,10 +477,7 @@ public final class ZkController {
       host = "http://" + hostaddress;
     } else {
       Matcher m = URL_PREFIX.matcher(host);
-      if (m.matches()) {
-        String prefix = m.group(1);
-        host = prefix + host;
-      } else {
+      if (!m.matches()) {
         host = "http://" + host;
       }
     }
@@ -1337,21 +1334,7 @@ public final class ZkController {
     // this also gets us our assigned shard id if it was not specified
     try {
       CloudDescriptor cloudDesc = cd.getCloudDescriptor();
-      if(cd.getCloudDescriptor().getCollectionName() !=null && cloudDesc.getCoreNodeName() != null ) {
-        //we were already registered
-        if(zkStateReader.getClusterState().hasCollection(cloudDesc.getCollectionName())){
-        DocCollection coll = zkStateReader.getClusterState().getCollection(cloudDesc.getCollectionName());
-         if(!"true".equals(coll.getStr("autoCreated"))){
-           Slice slice = coll.getSlice(cloudDesc.getShardId());
-           if(slice != null){
-             if(slice.getReplica(cloudDesc.getCoreNodeName()) == null) {
-               log.info("core_removed This core is removed from ZK");
-               throw new SolrException(ErrorCode.NOT_FOUND,cloudDesc.getCoreNodeName() +" is removed");
-             }
-           }
-         }
-        }
-      }
+
 
       // make sure the node name is set on the descriptor
       if (cloudDesc.getCoreNodeName() == null) {

Modified: lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java Sun Jan  5 21:04:58 2014
@@ -399,7 +399,7 @@ public class DefaultSolrHighlighter exte
         params.getBool( HighlightParams.USE_PHRASE_HIGHLIGHTER, true ),
         // FVH cannot process hl.requireFieldMatch parameter per-field basis
         params.getBool( HighlightParams.FIELD_MATCH, false ) );
-    fvh.setPhraseLimit(params.getInt(HighlightParams.PHRASE_LIMIT, Integer.MAX_VALUE));
+    fvh.setPhraseLimit(params.getInt(HighlightParams.PHRASE_LIMIT, SolrHighlighter.DEFAULT_PHRASE_LIMIT));
     FieldQuery fieldQuery = fvh.getFieldQuery( query, searcher.getIndexReader() );
 
     // Highlight each document
@@ -602,6 +602,10 @@ public class DefaultSolrHighlighter exte
     String alternateField = params.getFieldParam(fieldName, HighlightParams.ALTERNATE_FIELD);
     if (alternateField != null && alternateField.length() > 0) {
       StorableField[] docFields = doc.getFields(alternateField);
+      if (docFields.length == 0) {
+        // The alternate field did not exist, treat the original field as fallback instead
+        docFields = doc.getFields(fieldName);
+      }
       List<String> listFields = new ArrayList<String>();
       for (StorableField field : docFields) {
         if (field.binaryValue() == null)

Modified: lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/highlight/SolrHighlighter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/highlight/SolrHighlighter.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/highlight/SolrHighlighter.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/highlight/SolrHighlighter.java Sun Jan  5 21:04:58 2014
@@ -34,6 +34,8 @@ import org.apache.solr.util.SolrPluginUt
 
 public abstract class SolrHighlighter
 {
+
+  public static int DEFAULT_PHRASE_LIMIT = 5000;
   public static Logger log = LoggerFactory.getLogger(SolrHighlighter.class);
 
   @Deprecated

Modified: lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java Sun Jan  5 21:04:58 2014
@@ -17,7 +17,12 @@
 
 package org.apache.solr.search;
 
+import org.apache.lucene.queries.function.FunctionQuery;
+import org.apache.lucene.queries.function.FunctionValues;
+import org.apache.lucene.queries.function.ValueSource;
 import org.apache.lucene.util.BytesRef;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.request.LocalSolrQueryRequest;
 import org.apache.solr.schema.TrieFloatField;
 import org.apache.solr.schema.TrieIntField;
 import org.apache.solr.schema.TrieLongField;
@@ -47,6 +52,7 @@ import java.util.Arrays;
 import java.util.Map;
 import java.util.Set;
 import java.util.HashSet;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Iterator;
 
@@ -242,7 +248,7 @@ public class CollapsingQParserPlugin ext
         SchemaField schemaField = schema.getField(this.field);
 
         SortedDocValues docValues = null;
-
+        FunctionQuery funcQuery = null;
         if(schemaField.hasDocValues()) {
           docValues = searcher.getAtomicReader().getSortedDocValues(this.field);
         } else {
@@ -252,11 +258,39 @@ public class CollapsingQParserPlugin ext
         FieldType fieldType = null;
 
         if(this.max != null) {
-          fieldType = searcher.getSchema().getField(this.max).getType();
+          if(this.max.indexOf("(") == -1) {
+            fieldType = searcher.getSchema().getField(this.max).getType();
+          } else {
+            LocalSolrQueryRequest request = null;
+            try {
+              SolrParams params = new ModifiableSolrParams();
+              request = new LocalSolrQueryRequest(searcher.getCore(), params);
+              FunctionQParser functionQParser = new FunctionQParser(this.max, null, null,request);
+              funcQuery = (FunctionQuery)functionQParser.parse();
+            } catch (Exception e) {
+              throw new IOException(e);
+            } finally {
+              request.close();
+            }
+          }
         }
 
         if(this.min != null) {
-          fieldType = searcher.getSchema().getField(this.min).getType();
+          if(this.min.indexOf("(") == -1) {
+            fieldType = searcher.getSchema().getField(this.min).getType();
+          } else {
+            LocalSolrQueryRequest request = null;
+            try {
+              SolrParams params = new ModifiableSolrParams();
+              request = new LocalSolrQueryRequest(searcher.getCore(), params);
+              FunctionQParser functionQParser = new FunctionQParser(this.min, null, null,request);
+              funcQuery = (FunctionQuery)functionQParser.parse();
+            } catch (Exception e) {
+              throw new IOException(e);
+            } finally {
+              request.close();
+            }
+          }
         }
 
         int maxDoc = searcher.maxDoc();
@@ -274,7 +308,8 @@ public class CollapsingQParserPlugin ext
                                                    max != null,
                                                    this.needsScores,
                                                    fieldType,
-                                                   boostDocs);
+                                                   boostDocs,
+                                                   funcQuery);
         } else {
           return new CollapsingScoreCollector(maxDoc, leafCount, docValues, this.nullPolicy, boostDocs);
         }
@@ -508,7 +543,8 @@ public class CollapsingQParserPlugin ext
                                          boolean max,
                                          boolean needsScores,
                                          FieldType fieldType,
-                                         IntOpenHashSet boostDocs) throws IOException{
+                                         IntOpenHashSet boostDocs,
+                                         FunctionQuery funcQuery) throws IOException{
 
       this.maxDoc = maxDoc;
       this.contexts = new AtomicReaderContext[segments];
@@ -517,14 +553,18 @@ public class CollapsingQParserPlugin ext
       this.nullPolicy = nullPolicy;
       this.needsScores = needsScores;
       this.boostDocs = boostDocs;
-      if(fieldType instanceof TrieIntField) {
-        this.fieldValueCollapse = new IntValueCollapse(maxDoc, field, nullPolicy, new int[valueCount], max, this.needsScores, boostDocs);
-      } else if(fieldType instanceof TrieLongField) {
-        this.fieldValueCollapse =  new LongValueCollapse(maxDoc, field, nullPolicy, new int[valueCount], max, this.needsScores, boostDocs);
-      } else if(fieldType instanceof TrieFloatField) {
-        this.fieldValueCollapse =  new FloatValueCollapse(maxDoc, field, nullPolicy, new int[valueCount], max, this.needsScores, boostDocs);
+      if(funcQuery != null) {
+        this.fieldValueCollapse =  new ValueSourceCollapse(maxDoc, field, nullPolicy, new int[valueCount], max, this.needsScores, boostDocs, funcQuery);
       } else {
-        throw new IOException("min/max must be either TrieInt, TrieLong or TrieFloat.");
+        if(fieldType instanceof TrieIntField) {
+          this.fieldValueCollapse = new IntValueCollapse(maxDoc, field, nullPolicy, new int[valueCount], max, this.needsScores, boostDocs);
+        } else if(fieldType instanceof TrieLongField) {
+          this.fieldValueCollapse =  new LongValueCollapse(maxDoc, field, nullPolicy, new int[valueCount], max, this.needsScores, boostDocs);
+        } else if(fieldType instanceof TrieFloatField) {
+          this.fieldValueCollapse =  new FloatValueCollapse(maxDoc, field, nullPolicy, new int[valueCount], max, this.needsScores, boostDocs);
+        } else {
+          throw new IOException("min/max must be either TrieInt, TrieLong or TrieFloat.");
+        }
       }
     }
 
@@ -877,6 +917,97 @@ public class CollapsingQParserPlugin ext
     }
   }
 
+  private class ValueSourceCollapse extends FieldValueCollapse {
+
+    private FloatCompare comp;
+    private float nullVal;
+    private ValueSource valueSource;
+    private FunctionValues functionValues;
+    private float[] ordVals;
+    private Map rcontext = new HashMap();
+    private CollapseScore collapseScore = new CollapseScore();
+    private float score;
+    private boolean cscore;
+
+    public ValueSourceCollapse(int maxDoc,
+                               String funcStr,
+                               int nullPolicy,
+                               int[] ords,
+                               boolean max,
+                               boolean needsScores,
+                               IntOpenHashSet boostDocs,
+                               FunctionQuery funcQuery) throws IOException {
+      super(maxDoc, null, nullPolicy, max, needsScores, boostDocs);
+      this.valueSource = funcQuery.getValueSource();
+      this.ords = ords;
+      this.ordVals = new float[ords.length];
+      Arrays.fill(ords, -1);
+
+      if(max) {
+        comp = new MaxFloatComp();
+        Arrays.fill(ordVals, -Float.MAX_VALUE );
+      } else {
+        this.nullVal = Float.MAX_VALUE;
+        comp = new MinFloatComp();
+        Arrays.fill(ordVals, Float.MAX_VALUE);
+      }
+
+      if(funcStr.indexOf("cscore()") != -1) {
+        this.cscore = true;
+        this.rcontext.put("CSCORE",this.collapseScore);
+      }
+
+      if(this.needsScores) {
+        this.scores = new float[ords.length];
+        if(nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) {
+          nullScores = new FloatArrayList();
+        }
+      }
+    }
+
+    public void setNextReader(AtomicReaderContext context) throws IOException {
+      functionValues = this.valueSource.getValues(rcontext, context);
+    }
+
+    public void collapse(int ord, int contextDoc, int globalDoc) throws IOException {
+      if(needsScores || cscore) {
+        this.score = scorer.score();
+        this.collapseScore.score = score;
+      }
+
+      float val = functionValues.floatVal(contextDoc);
+
+      if(ord > -1) {
+        if(comp.test(val, ordVals[ord])) {
+          ords[ord] = globalDoc;
+          ordVals[ord] = val;
+          if(needsScores) {
+            scores[ord] = score;
+          }
+        }
+      } else if (this.collapsedSet.fastGet(globalDoc)) {
+        //Elevated doc so do nothing
+      } else if(this.nullPolicy == CollapsingPostFilter.NULL_POLICY_COLLAPSE) {
+        if(comp.test(val, nullVal)) {
+          nullVal = val;
+          nullDoc = globalDoc;
+          if(needsScores) {
+            nullScore = score;
+          }
+        }
+      } else if(this.nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) {
+        this.collapsedSet.fastSet(globalDoc);
+        if(needsScores) {
+          nullScores.add(score);
+        }
+      }
+    }
+  }
+
+  public static final class CollapseScore {
+    public float score;
+  }
+
   private interface IntCompare {
     public boolean test(int i1, int i2);
   }

Modified: lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java Sun Jan  5 21:04:58 2014
@@ -41,6 +41,7 @@ import org.apache.solr.common.util.Named
 import org.apache.solr.request.SolrRequestInfo;
 import org.apache.solr.schema.*;
 
+import org.apache.solr.search.function.CollapseScoreFunction;
 import org.apache.solr.search.function.distance.*;
 import org.apache.solr.util.plugin.NamedListInitializedPlugin;
 
@@ -221,6 +222,12 @@ public abstract class ValueSourceParser 
         };
       }
     });
+    addParser("cscore", new ValueSourceParser() {
+      @Override
+      public ValueSource parse(FunctionQParser fp) throws SyntaxError {
+        return new CollapseScoreFunction();
+      }
+    });
     addParser("sum", new ValueSourceParser() {
       @Override
       public ValueSource parse(FunctionQParser fp) throws SyntaxError {

Modified: lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/update/PeerSync.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/update/PeerSync.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/update/PeerSync.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/update/PeerSync.java Sun Jan  5 21:04:58 2014
@@ -291,7 +291,8 @@ public class PeerSync  {
       if (cantReachIsSuccess && sreq.purpose == 1 && srsp.getException() instanceof SolrServerException) {
         Throwable solrException = ((SolrServerException) srsp.getException())
             .getRootCause();
-        if (solrException instanceof ConnectException || solrException instanceof ConnectTimeoutException
+        boolean connectTimeoutExceptionInChain = connectTimeoutExceptionInChain(srsp.getException());
+        if (connectTimeoutExceptionInChain || solrException instanceof ConnectException || solrException instanceof ConnectTimeoutException
             || solrException instanceof NoHttpResponseException || solrException instanceof SocketException) {
           log.warn(msg() + " couldn't connect to " + srsp.getShardAddress() + ", counting as success");
 
@@ -309,6 +310,10 @@ public class PeerSync  {
             "Perhaps /get is not registered?");
         return true;
       }
+      
+      // TODO: we should return the above information so that when we can request a recovery through zookeeper, we do
+      // that for these nodes
+      
       // TODO: at least log???
       // srsp.getException().printStackTrace(System.out);
      
@@ -324,6 +329,23 @@ public class PeerSync  {
     }
   }
   
+  // sometimes the root exception is a SocketTimeoutException, but ConnectTimeoutException
+  // is in the chain
+  private boolean connectTimeoutExceptionInChain(Throwable exception) {
+    Throwable t = exception;
+    while (true) {
+      if (t instanceof ConnectTimeoutException) {
+        return true;
+      }
+      Throwable cause = t.getCause();
+      if (cause != null) {
+        t = cause;
+      } else {
+        return false;
+      }
+    }
+  }
+
   private boolean handleVersions(ShardResponse srsp) {
     // we retrieved the last N updates from the replica
     List<Long> otherVersions = (List<Long>)srsp.getSolrResponse().getResponse().get("versions");

Modified: lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java Sun Jan  5 21:04:58 2014
@@ -105,13 +105,8 @@ public class SolrIndexSplitter {
     // - need to worry about if IW.addIndexes does a sync or not...
     // - would be more efficient on the read side, but prob less efficient merging
 
-    IndexReader[] subReaders = new IndexReader[leaves.size()];
     for (int partitionNumber=0; partitionNumber<numPieces; partitionNumber++) {
-      log.info("SolrIndexSplitter: partition #" + partitionNumber + (ranges != null ? " range=" + ranges.get(partitionNumber) : ""));
-
-      for (int segmentNumber = 0; segmentNumber<subReaders.length; segmentNumber++) {
-        subReaders[segmentNumber] = new LiveDocsReader( leaves.get(segmentNumber), segmentDocSets.get(segmentNumber)[partitionNumber] );
-      }
+      log.info("SolrIndexSplitter: partition #" + partitionNumber + " partitionCount=" + numPieces + (ranges != null ? " range=" + ranges.get(partitionNumber) : ""));
 
       boolean success = false;
 
@@ -130,8 +125,12 @@ public class SolrIndexSplitter {
       }
 
       try {
-        // This merges the subreaders and will thus remove deletions (i.e. no optimize needed)
-        iw.addIndexes(subReaders);
+        // This removes deletions but optimize might still be needed because sub-shards will have the same number of segments as the parent shard.
+        for (int segmentNumber = 0; segmentNumber<leaves.size(); segmentNumber++) {
+          log.info("SolrIndexSplitter: partition #" + partitionNumber + " partitionCount=" + numPieces + (ranges != null ? " range=" + ranges.get(partitionNumber) : "") + " segment #"+segmentNumber + " segmentCount=" + leaves.size());
+          IndexReader subReader = new LiveDocsReader( leaves.get(segmentNumber), segmentDocSets.get(segmentNumber)[partitionNumber] );
+          iw.addIndexes(subReader);
+        }
         success = true;
       } finally {
         if (iwRef != null) {

Modified: lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java Sun Jan  5 21:04:58 2014
@@ -614,6 +614,12 @@ public class DistributedUpdateProcessor 
       // if one node is a RetryNode, this was a forward request
       if (errors.get(0).req.node instanceof RetryNode) {
         rsp.setException(errors.get(0).e);
+      } else {
+        if (log.isWarnEnabled()) {
+          for (Error error : errors) {
+            log.warn("Error sending update", error.e);
+          }
+        }
       }
       // else
       // for now we don't error - we assume if it was added locally, we
@@ -625,7 +631,6 @@ public class DistributedUpdateProcessor 
     // recover - the doc was already added locally, so it should have been
     // legit
 
-    // TODO: we should do this in the background it would seem
     for (final SolrCmdDistributor.Error error : errors) {
       if (error.req.node instanceof RetryNode) {
         // we don't try to force a leader to recover
@@ -1271,22 +1276,10 @@ public class DistributedUpdateProcessor 
       return;
     }
 
-    if (zkController.isConnected()) {
+    if (!zkController.getZkClient().getConnectionManager().isLikelyExpired()) {
       return;
     }
     
-    long timeoutAt = System.currentTimeMillis() + zkController.getClientTimeout();
-    while (System.currentTimeMillis() < timeoutAt) {
-      if (zkController.isConnected()) {
-        return;
-      }
-      try {
-        Thread.sleep(100);
-      } catch (InterruptedException e) {
-        Thread.currentThread().interrupt();
-        break;
-      }
-    }
     throw new SolrException(ErrorCode.SERVICE_UNAVAILABLE, "Cannot talk to ZooKeeper - Updates are disabled.");
   }
 

Modified: lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java Sun Jan  5 21:04:58 2014
@@ -282,14 +282,6 @@ public class BasicDistributedZk2Test ext
     
     // kill a shard
     CloudJettyRunner deadShard = chaosMonkey.stopShard(SHARD1, 0);
-
-
-    // we are careful to make sure the downed node is no longer in the state,
-    // because on some systems (especially freebsd w/ blackhole enabled), trying
-    // to talk to a downed node causes grief
-    Set<CloudJettyRunner> jetties = new HashSet<CloudJettyRunner>();
-    jetties.addAll(shardToJetty.get(SHARD1));
-    jetties.remove(deadShard);
     
     // ensure shard is dead
     try {

Modified: lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java Sun Jan  5 21:04:58 2014
@@ -34,6 +34,7 @@ import java.util.concurrent.Future;
 import java.util.concurrent.SynchronousQueue;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.lucene.util.LuceneTestCase.Slow;
@@ -100,6 +101,8 @@ public class BasicDistributedZkTest exte
   private String oneInstanceCollection = "oneInstanceCollection";
   private String oneInstanceCollection2 = "oneInstanceCollection2";
   
+  private AtomicInteger nodeCounter = new AtomicInteger();
+  
   ThreadPoolExecutor executor = new ThreadPoolExecutor(0,
       Integer.MAX_VALUE, 5, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),
       new DefaultSolrThreadFactory("testExecutor"));
@@ -928,11 +931,16 @@ public class BasicDistributedZkTest exte
         try {
           server = new HttpSolrServer(baseUrl);
           server.setConnectionTimeout(15000);
-          server.setSoTimeout(60000);
           Create createCmd = new Create();
           createCmd.setRoles("none");
           createCmd.setCoreName(collection + num);
           createCmd.setCollection(collection);
+          
+          if (random().nextBoolean()) {
+            // sometimes we use an explicit core node name
+            createCmd.setCoreNodeName("anode" + nodeCounter.incrementAndGet());
+          }
+          
           if (shardId == null) {
             createCmd.setNumShards(2);
           }

Modified: lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/DeleteInactiveReplicaTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/DeleteInactiveReplicaTest.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/DeleteInactiveReplicaTest.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/DeleteInactiveReplicaTest.java Sun Jan  5 21:04:58 2014
@@ -34,7 +34,9 @@ import org.apache.solr.common.params.Map
 import org.apache.solr.common.util.NamedList;
 import org.junit.After;
 import org.junit.Before;
+import org.junit.Ignore;
 
+@Ignore("Not currently valid see SOLR-5580")
 public class DeleteInactiveReplicaTest extends DeleteReplicaTest{
   private CloudSolrServer client;
 

Modified: lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionProcessorTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionProcessorTest.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionProcessorTest.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionProcessorTest.java Sun Jan  5 21:04:58 2014
@@ -218,6 +218,15 @@ public class OverseerCollectionProcessor
       }).anyTimes();
       
     }
+    
+    solrZkClientMock.getZkClientTimeout();
+    expectLastCall().andAnswer(new IAnswer<Object>() {
+      @Override
+      public Object answer() throws Throwable {
+        return 30000;
+      }
+    }).anyTimes();
+    
     clusterStateMock.hasCollection(anyObject(String.class));
     expectLastCall().andAnswer(new IAnswer<Boolean>() {
       @Override

Modified: lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java Sun Jan  5 21:04:58 2014
@@ -135,7 +135,7 @@ public class SyncSliceTest extends Abstr
     
     HttpSolrServer baseServer = new HttpSolrServer(baseUrl);
     // we only set the connect timeout, not so timeout
-    baseServer.setConnectionTimeout(15000);
+    baseServer.setConnectionTimeout(30000);
     baseServer.request(request);
     
     waitForThingsToLevelOut(15);
@@ -165,7 +165,7 @@ public class SyncSliceTest extends Abstr
 
     Thread.sleep(2000);
     
-    waitForThingsToLevelOut(90);
+    waitForThingsToLevelOut(120);
     
     Thread.sleep(1000);
     
@@ -228,7 +228,7 @@ public class SyncSliceTest extends Abstr
     
     Thread.sleep(3000);
     
-    waitForThingsToLevelOut(90);
+    waitForThingsToLevelOut(120);
     
     Thread.sleep(2000);
     

Modified: lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java Sun Jan  5 21:04:58 2014
@@ -33,6 +33,8 @@ import org.apache.solr.common.cloud.Solr
 import org.apache.solr.common.cloud.ZkNodeProps;
 import org.apache.solr.common.cloud.ZkStateReader;
 import org.apache.solr.util.ExternalPaths;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -262,6 +264,47 @@ public class ZkCLITest extends SolrTestC
     assertEquals(0, zkClient.getChildren("/", null, true).size());
   }
   
+  @Test
+  public void testGet() throws Exception {
+    String getNode = "/getNode";
+    byte [] data = new String("getNode-data").getBytes("UTF-8");
+    this.zkClient.create(getNode, data, CreateMode.PERSISTENT, true);
+    String[] args = new String[] {"-zkhost", zkServer.getZkAddress(), "-cmd",
+        "get", getNode};
+    ZkCLI.main(args);
+  }
+
+  @Test
+  public void testGetFile() throws Exception {
+    String getNode = "/getFileNode";
+    byte [] data = new String("getFileNode-data").getBytes("UTF-8");
+    this.zkClient.create(getNode, data, CreateMode.PERSISTENT, true);
+
+    File file = new File(TEMP_DIR,
+        "solrtest-getfile-" + this.getClass().getName() + "-" + System.currentTimeMillis());
+    String[] args = new String[] {"-zkhost", zkServer.getZkAddress(), "-cmd",
+        "getfile", getNode, file.getAbsolutePath()};
+    ZkCLI.main(args);
+
+    byte [] readData = FileUtils.readFileToByteArray(file);
+    assertArrayEquals(data, readData);
+  }
+
+  @Test
+  public void testGetFileNotExists() throws Exception {
+    String getNode = "/getFileNotExistsNode";
+
+    File file = new File(TEMP_DIR,
+        "solrtest-getfilenotexists-" + this.getClass().getName() + "-" + System.currentTimeMillis());
+    String[] args = new String[] {"-zkhost", zkServer.getZkAddress(), "-cmd",
+        "getfile", getNode, file.getAbsolutePath()};
+    try {
+      ZkCLI.main(args);
+      fail("Expected NoNodeException");
+    } catch (KeeperException.NoNodeException ex) {
+    }
+  }
+
   @Override
   public void tearDown() throws Exception {
     if (VERBOSE) {

Modified: lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java Sun Jan  5 21:04:58 2014
@@ -32,9 +32,11 @@ import org.junit.BeforeClass;
 import org.junit.Test;
 
 import java.io.File;
+import java.io.IOException;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.TimeoutException;
 
 @Slow
 public class ZkControllerTest extends SolrTestCaseJ4 {
@@ -255,6 +257,46 @@ public class ZkControllerTest extends So
 
   }
 
+  @Test
+  public void testGetHostName() throws Exception {
+    String zkDir = dataDir.getAbsolutePath() + File.separator
+        + "zookeeper/server1/data";
+    CoreContainer cc = null;
+
+    ZkTestServer server = new ZkTestServer(zkDir);
+    try {
+      server.run();
+
+      AbstractZkTestCase.tryCleanSolrZkNode(server.getZkHost());
+      AbstractZkTestCase.makeSolrZkNode(server.getZkHost());
+
+      cc = getCoreContainer();
+      ZkController zkController = null;
+
+      try {
+        zkController = new ZkController(cc, server.getZkAddress(), TIMEOUT, 10000,
+            "http://127.0.0.1", "8983", "solr", 0, true, new CurrentCoreDescriptorProvider() {
+
+          @Override
+          public List<CoreDescriptor> getCurrentDescriptors() {
+            // do nothing
+            return null;
+          }
+        });
+      } catch (IllegalArgumentException e) {
+        fail("ZkController did not normalize host name correctly");
+      } finally {
+        if (zkController != null)
+          zkController.close();
+      }
+    } finally {
+      if (cc != null) {
+        cc.shutdown();
+      }
+      server.shutdown();
+    }
+  }
+
   private CoreContainer getCoreContainer() {
     CoreContainer cc = new CoreContainer(solrHomeDirectory.getAbsolutePath());
     cc.load();

Modified: lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java Sun Jan  5 21:04:58 2014
@@ -56,6 +56,8 @@ public class HdfsTestUtil {
     System.setProperty("test.cache.data", dir.getAbsolutePath() + File.separator + "hdfs" + File.separator + "cache");
     System.setProperty("solr.lock.type", "hdfs");
     
+    System.setProperty("solr.hdfs.home", "/solr_hdfs_home");
+    
     MiniDFSCluster dfsCluster = new MiniDFSCluster(conf, dataNodes, true, null);
     
     SolrTestCaseJ4.useFactory("org.apache.solr.core.HdfsDirectoryFactory");
@@ -68,6 +70,7 @@ public class HdfsTestUtil {
     System.clearProperty("solr.lock.type");
     System.clearProperty("test.build.data");
     System.clearProperty("test.cache.data");
+    System.clearProperty("solr.hdfs.home");
     if (dfsCluster != null) {
       dfsCluster.shutdown();
     }

Modified: lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/hdfs/StressHdfsTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/hdfs/StressHdfsTest.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/hdfs/StressHdfsTest.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/hdfs/StressHdfsTest.java Sun Jan  5 21:04:58 2014
@@ -128,7 +128,6 @@ public class StressHdfsTest extends Basi
         }
         
         c.query(new SolrQuery("id:" + i));
-        c.setSoTimeout(60000);
         c.setConnectionTimeout(30000);
         NamedList<Object> response = c.query(
             new SolrQuery().setRequestHandler("/admin/system")).getResponse();

Modified: lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/highlight/HighlighterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/highlight/HighlighterTest.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/highlight/HighlighterTest.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/highlight/HighlighterTest.java Sun Jan  5 21:04:58 2014
@@ -660,6 +660,16 @@ public class HighlighterTest extends Sol
             "//lst[@name='highlighting']/lst[@name='1' and count(*)=1]",
             "//lst[@name='highlighting']/lst[@name='1']/arr[@name='t_text']/str[.='a piece of text']"
             );
+
+    // with a non-existing alternate field + max length
+    args.put("hl.alternateField", "NonExistingField");
+    args.put("hl.maxAlternateFieldLength", "15");
+    sumLRF = h.getRequestFactory("standard", 0, 200, args);
+    assertQ("Alternate summarization",
+            sumLRF.makeRequest("tv_text:keyword"),
+            "//lst[@name='highlighting']/lst[@name='1' and count(*)=1]",
+            "//lst[@name='highlighting']/lst[@name='1']/arr[@name='t_text']/str[.='a piece of text']"
+            );
   }
   
   @Test

Modified: lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java Sun Jan  5 21:04:58 2014
@@ -343,6 +343,11 @@ public class QueryEqualityTest extends S
   public void testFuncRord() throws Exception {
     assertFuncEquals("rord(foo_s)","rord(foo_s    )"); 
   }
+
+  public void testFuncCscore() throws Exception {
+    assertFuncEquals("cscore()", "cscore(  )");
+  }
+
   public void testFuncTop() throws Exception {
     assertFuncEquals("top(sum(3,foo_i))");
   }

Modified: lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/search/TestCollapseQParserPlugin.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/search/TestCollapseQParserPlugin.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/search/TestCollapseQParserPlugin.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/search/TestCollapseQParserPlugin.java Sun Jan  5 21:04:58 2014
@@ -95,6 +95,40 @@ public class TestCollapseQParserPlugin e
         "//result/doc[4]/float[@name='id'][.='6.0']"
     );
 
+    // Test value source collapse criteria
+    params = new ModifiableSolrParams();
+    params.add("q", "*:*");
+    params.add("fq", "{!collapse field=group_s nullPolicy=collapse min=field(test_ti)}");
+    params.add("sort", "test_ti desc");
+    assertQ(req(params), "*[count(//doc)=3]",
+        "//result/doc[1]/float[@name='id'][.='4.0']",
+        "//result/doc[2]/float[@name='id'][.='1.0']",
+        "//result/doc[3]/float[@name='id'][.='5.0']"
+    );
+
+    // Test value source collapse criteria with cscore function
+    params = new ModifiableSolrParams();
+    params.add("q", "*:*");
+    params.add("fq", "{!collapse field=group_s nullPolicy=collapse min=cscore()}");
+    params.add("defType", "edismax");
+    params.add("bf", "field(test_ti)");
+    assertQ(req(params), "*[count(//doc)=3]",
+        "//result/doc[1]/float[@name='id'][.='4.0']",
+        "//result/doc[2]/float[@name='id'][.='1.0']",
+        "//result/doc[3]/float[@name='id'][.='5.0']"
+    );
+
+    // Test value source collapse criteria with compound cscore function
+    params = new ModifiableSolrParams();
+    params.add("q", "*:*");
+    params.add("fq", "{!collapse field=group_s nullPolicy=collapse min=sum(cscore(),field(test_ti))}");
+    params.add("defType", "edismax");
+    params.add("bf", "field(test_ti)");
+    assertQ(req(params), "*[count(//doc)=3]",
+        "//result/doc[1]/float[@name='id'][.='4.0']",
+        "//result/doc[2]/float[@name='id'][.='1.0']",
+        "//result/doc[3]/float[@name='id'][.='5.0']"
+    );
 
     //Test collapse by score with elevation
 

Modified: lucene/dev/branches/lucene5376/solr/example/solr/collection1/conf/schema.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/example/solr/collection1/conf/schema.xml?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/example/solr/collection1/conf/schema.xml (original)
+++ lucene/dev/branches/lucene5376/solr/example/solr/collection1/conf/schema.xml Sun Jan  5 21:04:58 2014
@@ -779,6 +779,18 @@
       </analyzer>
     </fieldType>
 
+    <!-- Kurdish -->
+    <fieldType name="text_ckb" class="solr.TextField" positionIncrementGap="100">
+      <analyzer>
+        <tokenizer class="solr.StandardTokenizerFactory"/>
+        <filter class="solr.SoraniNormalizationFilterFactory"/>
+        <!-- for any latin text -->
+        <filter class="solr.LowerCaseFilterFactory"/>
+        <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_ckb.txt"/>
+        <filter class="solr.SoraniStemFilterFactory"/>
+      </analyzer>
+    </fieldType>
+
     <!-- Czech -->
     <fieldType name="text_cz" class="solr.TextField" positionIncrementGap="100">
       <analyzer> 

Modified: lucene/dev/branches/lucene5376/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrServer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrServer.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrServer.java (original)
+++ lucene/dev/branches/lucene5376/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrServer.java Sun Jan  5 21:04:58 2014
@@ -384,7 +384,7 @@ public class ConcurrentUpdateSolrServer 
         synchronized (runners) {
           runner = runners.peek();
         }
-        if (runner == null)
+        if (runner == null || scheduler.isTerminated())
           break;
         runner.runnerLock.lock();
         runner.runnerLock.unlock();



Mime
View raw message