lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rm...@apache.org
Subject svn commit: r1440839 [2/3] - in /lucene/dev/branches/lucene4547: ./ dev-tools/ dev-tools/idea/lucene/demo/ dev-tools/maven/lucene/demo/ lucene/ lucene/core/ lucene/core/src/java/org/apache/lucene/codecs/compressing/ lucene/core/src/java/org/apache/luce...
Date Thu, 31 Jan 2013 03:37:34 GMT
Modified: lucene/dev/branches/lucene4547/lucene/facet/src/test/org/apache/lucene/facet/search/TestSameRequestAccumulation.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/facet/src/test/org/apache/lucene/facet/search/TestSameRequestAccumulation.java?rev=1440839&r1=1440838&r2=1440839&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/facet/src/test/org/apache/lucene/facet/search/TestSameRequestAccumulation.java (original)
+++ lucene/dev/branches/lucene4547/lucene/facet/src/test/org/apache/lucene/facet/search/TestSameRequestAccumulation.java Thu Jan 31 03:37:31 2013
@@ -3,6 +3,7 @@ package org.apache.lucene.facet.search;
 import java.util.List;
 
 import org.apache.lucene.facet.FacetTestBase;
+import org.apache.lucene.facet.index.params.FacetIndexingParams;
 import org.apache.lucene.facet.search.FacetsCollector;
 import org.apache.lucene.facet.search.params.CountFacetRequest;
 import org.apache.lucene.facet.search.params.FacetSearchParams;
@@ -31,18 +32,21 @@ import org.junit.Before;
 
 public class TestSameRequestAccumulation extends FacetTestBase {
   
+  private FacetIndexingParams fip;
+  
   @Override
   @Before
   public void setUp() throws Exception {
     super.setUp();
-    initIndex();
+    fip = getFacetIndexingParams(Integer.MAX_VALUE);
+    initIndex(fip);
   }
   
   // Following LUCENE-4461 - ensure requesting the (exact) same request more
   // than once does not alter the results
   public void testTwoSameRequests() throws Exception {
     final CountFacetRequest facetRequest = new CountFacetRequest(new CategoryPath("root"), 10);
-    FacetSearchParams fsp = new FacetSearchParams(facetRequest);
+    FacetSearchParams fsp = new FacetSearchParams(fip, facetRequest);
     
     FacetsCollector fc = FacetsCollector.create(fsp, indexReader, taxoReader);
     searcher.search(new MatchAllDocsQuery(), fc);
@@ -50,7 +54,7 @@ public class TestSameRequestAccumulation
     final String expected = fc.getFacetResults().get(0).toString();
 
     // now add the same facet request with duplicates (same instance and same one)
-    fsp = new FacetSearchParams(facetRequest, facetRequest, new CountFacetRequest(new CategoryPath("root"), 10));
+    fsp = new FacetSearchParams(fip, facetRequest, facetRequest, new CountFacetRequest(new CategoryPath("root"), 10));
 
     // make sure the search params holds 3 requests now
     assertEquals(3, fsp.facetRequests.size());

Modified: lucene/dev/branches/lucene4547/lucene/facet/src/test/org/apache/lucene/facet/search/TestScoredDocIdCollector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/facet/src/test/org/apache/lucene/facet/search/TestScoredDocIdCollector.java?rev=1440839&r1=1440838&r2=1440839&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/facet/src/test/org/apache/lucene/facet/search/TestScoredDocIdCollector.java (original)
+++ lucene/dev/branches/lucene4547/lucene/facet/src/test/org/apache/lucene/facet/search/TestScoredDocIdCollector.java Thu Jan 31 03:37:31 2013
@@ -5,6 +5,7 @@ import java.util.Arrays;
 import java.util.List;
 
 import org.apache.lucene.facet.FacetTestBase;
+import org.apache.lucene.facet.index.params.FacetIndexingParams;
 import org.apache.lucene.facet.search.params.CountFacetRequest;
 import org.apache.lucene.facet.search.params.FacetSearchParams;
 import org.apache.lucene.facet.search.params.ScoreFacetRequest;
@@ -37,11 +38,14 @@ import org.junit.Test;
 /** Test ScoredDocIdCollector. */
 public class TestScoredDocIdCollector extends FacetTestBase {
 
+  private FacetIndexingParams fip;
+  
   @Override
   @Before
   public void setUp() throws Exception {
     super.setUp();
-    initIndex();
+    fip = getFacetIndexingParams(Integer.MAX_VALUE);
+    initIndex(fip);
   }
 
   @Override
@@ -73,8 +77,8 @@ public class TestScoredDocIdCollector ex
 
     // verify by facet values
     CategoryPath cp = new CategoryPath("root","a");
-    FacetSearchParams countFSP = new FacetSearchParams(getFacetIndexingParams(Integer.MAX_VALUE), new CountFacetRequest(cp, 10));
-    FacetSearchParams scoreFSP = new FacetSearchParams(getFacetIndexingParams(Integer.MAX_VALUE), new ScoreFacetRequest(cp, 10));
+    FacetSearchParams countFSP = new FacetSearchParams(fip, new CountFacetRequest(cp, 10));
+    FacetSearchParams scoreFSP = new FacetSearchParams(fip, new ScoreFacetRequest(cp, 10));
     
     List<FacetResult> countRes = findFacets(scoredDocIDs, countFSP);
     List<FacetResult> scoreRes = findFacets(scoredDocIDs, scoreFSP);
@@ -101,10 +105,8 @@ public class TestScoredDocIdCollector ex
   }
 
   // compute facets with certain facet requests and docs
-  private List<FacetResult> findFacets(ScoredDocIDs sDocids,
-      FacetSearchParams facetSearchParams) throws IOException {
-    FacetsAccumulator fAccumulator = new StandardFacetsAccumulator(
-        facetSearchParams, indexReader, taxoReader);
+  private List<FacetResult> findFacets(ScoredDocIDs sDocids, FacetSearchParams facetSearchParams) throws IOException {
+    FacetsAccumulator fAccumulator = new StandardFacetsAccumulator(facetSearchParams, indexReader, taxoReader);
     List<FacetResult> res = fAccumulator.accumulate(sDocids);
 
     // Results are ready, printing them...

Modified: lucene/dev/branches/lucene4547/lucene/facet/src/test/org/apache/lucene/facet/search/TestStandardFacetsAccumulator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/facet/src/test/org/apache/lucene/facet/search/TestStandardFacetsAccumulator.java?rev=1440839&r1=1440838&r2=1440839&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/facet/src/test/org/apache/lucene/facet/search/TestStandardFacetsAccumulator.java (original)
+++ lucene/dev/branches/lucene4547/lucene/facet/src/test/org/apache/lucene/facet/search/TestStandardFacetsAccumulator.java Thu Jan 31 03:37:31 2013
@@ -113,7 +113,6 @@ public class TestStandardFacetsAccumulat
     List<FacetResult> results = fc.getFacetResults();
     assertEquals("received too many facet results", 1, results.size());
     FacetResultNode frn = results.get(0).getFacetResultNode();
-    assertEquals("wrong weight for \"A\"", 4, (int) frn.value);
     assertEquals("wrong number of children", 2, frn.subResults.size());
     for (FacetResultNode node : frn.subResults) {
       assertEquals("wrong weight for child " + node.label, 2, (int) node.value);

Modified: lucene/dev/branches/lucene4547/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKInEachNodeResultHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKInEachNodeResultHandler.java?rev=1440839&r1=1440838&r2=1440839&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKInEachNodeResultHandler.java (original)
+++ lucene/dev/branches/lucene4547/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKInEachNodeResultHandler.java Thu Jan 31 03:37:31 2013
@@ -181,7 +181,6 @@ public class TestTopKInEachNodeResultHan
       boolean hasDoctor = "Doctor".equals(fr.getFacetRequest().categoryPath.components[0]);
       assertEquals(9, fr.getNumValidDescendants());
       FacetResultNode parentRes = fr.getFacetResultNode();
-      assertEquals(16.0, parentRes.value, Double.MIN_VALUE);
       assertEquals(2, parentRes.subResults.size());
       // two nodes sorted by descending values: a/b with 8  and a/c with 6
       // a/b has two children a/b/2 with value 3, and a/b/1 with value 2. 
@@ -217,7 +216,6 @@ public class TestTopKInEachNodeResultHan
       hasDoctor |= "Doctor".equals(fr.getFacetRequest().categoryPath.components[0]);
       assertEquals(9, fr.getNumValidDescendants());
       parentRes = fr.getFacetResultNode();
-      assertEquals(16.0, parentRes.value, Double.MIN_VALUE);
       assertEquals(2, parentRes.subResults.size());
       // two nodes sorted by descending values: a/b with 8  and a/c with 6
       // a/b has two children a/b/2 with value 3, and a/b/1 with value 2. 
@@ -234,7 +232,6 @@ public class TestTopKInEachNodeResultHan
       hasDoctor |= "Doctor".equals(fr.getFacetRequest().categoryPath.components[0]);
       assertEquals(4, fr.getNumValidDescendants(), 4);
       parentRes = fr.getFacetResultNode();
-      assertEquals(16.0, parentRes.value, Double.MIN_VALUE);
       assertEquals(2, parentRes.subResults.size());
       // two nodes sorted by descending values: 
       // a/b with value 8 and a/c with value 6

Modified: lucene/dev/branches/lucene4547/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKResultsHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKResultsHandler.java?rev=1440839&r1=1440838&r2=1440839&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKResultsHandler.java (original)
+++ lucene/dev/branches/lucene4547/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKResultsHandler.java Thu Jan 31 03:37:31 2013
@@ -4,6 +4,8 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 
+import org.apache.lucene.facet.index.params.CategoryListParams.OrdinalPolicy;
+import org.apache.lucene.facet.index.params.FacetIndexingParams;
 import org.apache.lucene.facet.search.params.CountFacetRequest;
 import org.apache.lucene.facet.search.params.FacetRequest;
 import org.apache.lucene.facet.search.params.FacetRequest.ResultMode;
@@ -73,7 +75,9 @@ public class TestTopKResultsHandler exte
   @Test
   public void testSimple() throws Exception {
     for (int partitionSize : partitionSizes) {
-      initIndex(partitionSize);
+      FacetIndexingParams fip = getFacetIndexingParams(partitionSize);
+      OrdinalPolicy op = fip.getCategoryListParams(null).getOrdinalPolicy(null);
+      initIndex(fip);
 
       List<FacetRequest> facetRequests = new ArrayList<FacetRequest>();
       facetRequests.add(new CountFacetRequest(new CategoryPath("a"), 100));
@@ -87,8 +91,8 @@ public class TestTopKResultsHandler exte
       facetRequests.add(new CountFacetRequest(new CategoryPath("a", "c"), 100));
       
       // do different facet counts and compare to control
-      FacetSearchParams sParams = getFacetSearchParams(facetRequests, getFacetIndexingParams(partitionSize));
-
+      FacetSearchParams sParams = getFacetSearchParams(facetRequests, fip);
+      
       FacetsCollector fc = new StandardFacetsCollector(sParams, indexReader, taxoReader) {
         @Override
         protected FacetsAccumulator initFacetsAccumulator(FacetSearchParams facetSearchParams, IndexReader indexReader, TaxonomyReader taxonomyReader) {
@@ -100,17 +104,21 @@ public class TestTopKResultsHandler exte
       
       searcher.search(new MatchAllDocsQuery(), fc);
       List<FacetResult> facetResults = fc.getFacetResults();
-
+      
       FacetResult fr = facetResults.get(0);
       FacetResultNode parentRes = fr.getFacetResultNode();
-      assertEquals(13.0, parentRes.value, Double.MIN_VALUE);
+      if (op == OrdinalPolicy.ALL_PARENTS) {
+        assertEquals(13.0, parentRes.value, Double.MIN_VALUE);
+      }
       FacetResultNode[] frn = resultNodesAsArray(parentRes);
       assertEquals(7.0, frn[0].value, Double.MIN_VALUE);
       assertEquals(6.0, frn[1].value, Double.MIN_VALUE);
 
       fr = facetResults.get(1);
       parentRes = fr.getFacetResultNode();
-      assertEquals(13.0, parentRes.value, Double.MIN_VALUE);
+      if (op == OrdinalPolicy.ALL_PARENTS) {
+        assertEquals(13.0, parentRes.value, Double.MIN_VALUE);
+      }
       frn = resultNodesAsArray(parentRes);
       assertEquals(7.0, frn[0].value, Double.MIN_VALUE);
       assertEquals(6.0, frn[1].value, Double.MIN_VALUE);
@@ -121,7 +129,9 @@ public class TestTopKResultsHandler exte
 
       fr = facetResults.get(2);
       parentRes = fr.getFacetResultNode();
-      assertEquals(7.0, parentRes.value, Double.MIN_VALUE);
+      if (op == OrdinalPolicy.ALL_PARENTS) {
+        assertEquals(7.0, parentRes.value, Double.MIN_VALUE);
+      }
       frn = resultNodesAsArray(parentRes);
       assertEquals(2.0, frn[0].value, Double.MIN_VALUE);
       assertEquals(2.0, frn[1].value, Double.MIN_VALUE);
@@ -130,13 +140,17 @@ public class TestTopKResultsHandler exte
 
       fr = facetResults.get(3);
       parentRes = fr.getFacetResultNode();
-      assertEquals(2.0, parentRes.value, Double.MIN_VALUE);
+      if (op == OrdinalPolicy.ALL_PARENTS) {
+        assertEquals(2.0, parentRes.value, Double.MIN_VALUE);
+      }
       frn = resultNodesAsArray(parentRes);
       assertEquals(0, frn.length);
 
       fr = facetResults.get(4);
       parentRes = fr.getFacetResultNode();
-      assertEquals(6.0, parentRes.value, Double.MIN_VALUE);
+      if (op == OrdinalPolicy.ALL_PARENTS) {
+        assertEquals(6.0, parentRes.value, Double.MIN_VALUE);
+      }
       frn = resultNodesAsArray(parentRes);
       assertEquals(1.0, frn[0].value, Double.MIN_VALUE);
       closeAll();
@@ -149,12 +163,12 @@ public class TestTopKResultsHandler exte
   @Test
   public void testGetMaxIntFacets() throws Exception {
     for (int partitionSize : partitionSizes) {
-      initIndex(partitionSize);
+      FacetIndexingParams fip = getFacetIndexingParams(partitionSize);
+      initIndex(fip);
 
       // do different facet counts and compare to control
       CategoryPath path = new CategoryPath("a", "b");
-      FacetSearchParams sParams = getFacetSearchParams(getFacetIndexingParams(partitionSize), 
-          new CountFacetRequest(path, Integer.MAX_VALUE));
+      FacetSearchParams sParams = getFacetSearchParams(fip, new CountFacetRequest(path, Integer.MAX_VALUE));
 
       FacetsCollector fc = new StandardFacetsCollector(sParams, indexReader, taxoReader) {
         @Override
@@ -174,7 +188,7 @@ public class TestTopKResultsHandler exte
 
       // As a control base results, ask for top-1000 results
       FacetSearchParams sParams2 = getFacetSearchParams(
-          getFacetIndexingParams(partitionSize), new CountFacetRequest(path, Integer.MAX_VALUE));
+          fip, new CountFacetRequest(path, Integer.MAX_VALUE));
 
       FacetsCollector fc2 = new StandardFacetsCollector(sParams2, indexReader, taxoReader) {
         @Override
@@ -207,12 +221,11 @@ public class TestTopKResultsHandler exte
   @Test
   public void testSimpleSearchForNonexistentFacet() throws Exception {
     for (int partitionSize : partitionSizes) {
-      initIndex(partitionSize);
+      FacetIndexingParams fip = getFacetIndexingParams(partitionSize);
+      initIndex(fip);
 
       CategoryPath path = new CategoryPath("Miau Hattulla");
-      FacetSearchParams sParams = getFacetSearchParams(
-          getFacetIndexingParams(partitionSize),
-          new CountFacetRequest(path, 10));
+      FacetSearchParams sParams = getFacetSearchParams(fip, new CountFacetRequest(path, 10));
 
       FacetsCollector fc = FacetsCollector.create(sParams, indexReader, taxoReader);
       

Modified: lucene/dev/branches/lucene4547/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKResultsHandlerRandom.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKResultsHandlerRandom.java?rev=1440839&r1=1440838&r2=1440839&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKResultsHandlerRandom.java (original)
+++ lucene/dev/branches/lucene4547/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKResultsHandlerRandom.java Thu Jan 31 03:37:31 2013
@@ -4,6 +4,7 @@ import java.io.IOException;
 import java.util.HashMap;
 import java.util.List;
 
+import org.apache.lucene.facet.index.params.FacetIndexingParams;
 import org.apache.lucene.facet.search.params.FacetSearchParams;
 import org.apache.lucene.facet.search.results.FacetResult;
 import org.apache.lucene.facet.search.results.FacetResultNode;
@@ -32,10 +33,10 @@ import org.junit.Test;
 
 public class TestTopKResultsHandlerRandom extends BaseTestTopK {
   
-  private List<FacetResult> countFacets(int partitionSize, int numResults, final boolean doComplement)
+  private List<FacetResult> countFacets(FacetIndexingParams fip, int numResults, final boolean doComplement)
       throws IOException {
     Query q = new MatchAllDocsQuery();
-    FacetSearchParams facetSearchParams = searchParamsWithRequests(numResults, partitionSize);
+    FacetSearchParams facetSearchParams = searchParamsWithRequests(numResults, fip);
     FacetsCollector fc = new StandardFacetsCollector(facetSearchParams, indexReader, taxoReader) {
       @Override
       protected FacetsAccumulator initFacetsAccumulator(
@@ -59,7 +60,8 @@ public class TestTopKResultsHandlerRando
   @Test
   public void testTopCountsOrder() throws Exception {
     for (int partitionSize : partitionSizes) {
-      initIndex(partitionSize);
+      FacetIndexingParams fip = getFacetIndexingParams(partitionSize);
+      initIndex(fip);
       
       /*
        * Try out faceted search in it's most basic form (no sampling nor complement
@@ -67,7 +69,7 @@ public class TestTopKResultsHandlerRando
        * being indexed, and later on an "over-all" faceted search is performed. The
        * results are checked against the DF of each facet by itself
        */
-      List<FacetResult> facetResults = countFacets(partitionSize, 100000, false);
+      List<FacetResult> facetResults = countFacets(fip, 100000, false);
       assertCountsAndCardinality(facetCountsTruth(), facetResults);
       
       /*
@@ -77,10 +79,10 @@ public class TestTopKResultsHandlerRando
        * place in here. The results are checked against the a regular (a.k.a
        * no-complement, no-sampling) faceted search with the same parameters.
        */
-      facetResults = countFacets(partitionSize, 100000, true);
+      facetResults = countFacets(fip, 100000, true);
       assertCountsAndCardinality(facetCountsTruth(), facetResults);
       
-      List<FacetResult> allFacetResults = countFacets(partitionSize, 100000, false);
+      List<FacetResult> allFacetResults = countFacets(fip, 100000, false);
       
       HashMap<String,Integer> all = new HashMap<String,Integer>();
       int maxNumNodes = 0;
@@ -108,7 +110,7 @@ public class TestTopKResultsHandlerRando
         if (VERBOSE) {
           System.out.println("-------  verify for "+n+" top results");
         }
-        List<FacetResult> someResults = countFacets(partitionSize, n, false);
+        List<FacetResult> someResults = countFacets(fip, n, false);
         k = 0;
         for (FacetResult fr : someResults) {
           FacetResultNode topResNode = fr.getFacetResultNode();

Modified: lucene/dev/branches/lucene4547/lucene/facet/src/test/org/apache/lucene/facet/search/TestTotalFacetCounts.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/facet/src/test/org/apache/lucene/facet/search/TestTotalFacetCounts.java?rev=1440839&r1=1440838&r2=1440839&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/facet/src/test/org/apache/lucene/facet/search/TestTotalFacetCounts.java (original)
+++ lucene/dev/branches/lucene4547/lucene/facet/src/test/org/apache/lucene/facet/search/TestTotalFacetCounts.java Thu Jan 31 03:37:31 2013
@@ -3,12 +3,20 @@ package org.apache.lucene.facet.search;
 import java.io.File;
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.Collections;
 
+import org.apache.lucene.document.Document;
 import org.apache.lucene.facet.FacetTestCase;
-import org.apache.lucene.facet.FacetTestUtils;
-import org.apache.lucene.facet.FacetTestUtils.IndexTaxonomyReaderPair;
-import org.apache.lucene.facet.FacetTestUtils.IndexTaxonomyWriterPair;
+import org.apache.lucene.facet.index.FacetFields;
+import org.apache.lucene.facet.index.params.CategoryListParams;
 import org.apache.lucene.facet.index.params.FacetIndexingParams;
+import org.apache.lucene.facet.taxonomy.CategoryPath;
+import org.apache.lucene.facet.taxonomy.TaxonomyReader;
+import org.apache.lucene.facet.taxonomy.TaxonomyWriter;
+import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader;
+import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util._TestUtil;
@@ -33,9 +41,9 @@ import org.junit.Test;
 
 public class TestTotalFacetCounts extends FacetTestCase {
 
-  private static void initCache(int numEntries) {
+  private static void initCache() {
     TotalFacetCountsCache.getSingleton().clear();
-    TotalFacetCountsCache.getSingleton().setCacheSize(numEntries); // Set to keep one in mem
+    TotalFacetCountsCache.getSingleton().setCacheSize(1); // Set to keep one in mem
   }
 
   @Test
@@ -48,51 +56,60 @@ public class TestTotalFacetCounts extend
   }
 
   private void doTestWriteRead(final int partitionSize) throws IOException {
-    initCache(1);
+    initCache();
 
-    // Create temporary RAMDirectories
-    Directory[][] dirs = FacetTestUtils.createIndexTaxonomyDirs(1);
-    // Create our index/taxonomy writers
-    IndexTaxonomyWriterPair[] writers = FacetTestUtils.createIndexTaxonomyWriterPair(dirs);
+    Directory indexDir = newDirectory();
+    Directory taxoDir = newDirectory();
+    IndexWriter indexWriter = new IndexWriter(indexDir, newIndexWriterConfig(TEST_VERSION_CURRENT, null));
+    TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
+    
     FacetIndexingParams iParams = new FacetIndexingParams() {
       @Override
       public int getPartitionSize() {
         return partitionSize;
       }
+      
+      @Override
+      public CategoryListParams getCategoryListParams(CategoryPath category) {
+        return new CategoryListParams() {
+          @Override
+          public OrdinalPolicy getOrdinalPolicy(String dimension) {
+            return OrdinalPolicy.ALL_PARENTS;
+          }
+        };
+      }
     };
     // The counts that the TotalFacetCountsArray should have after adding
     // the below facets to the index.
     int[] expectedCounts = new int[] { 0, 3, 1, 3, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1 };
-    
-    // Add a facet to the index
-    TestTotalFacetCountsCache.addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "a", "b");
-    TestTotalFacetCountsCache.addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "c", "d");
-    TestTotalFacetCountsCache.addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "a", "e");
-    TestTotalFacetCountsCache.addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "a", "d");
-    TestTotalFacetCountsCache.addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "c", "g");
-    TestTotalFacetCountsCache.addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "c", "z");
-    TestTotalFacetCountsCache.addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "b", "a");
-    TestTotalFacetCountsCache.addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "1", "2");
-    TestTotalFacetCountsCache.addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "b", "c");
+    String[] categories = new String[] { "a/b", "c/d", "a/e", "a/d", "c/g", "c/z", "b/a", "1/2", "b/c" };
+
+    FacetFields facetFields = new FacetFields(taxoWriter, iParams);
+    for (String cat : categories) {
+      Document doc = new Document();
+      facetFields.addFields(doc, Collections.singletonList(new CategoryPath(cat, '/')));
+      indexWriter.addDocument(doc);
+    }
 
     // Commit Changes
-    writers[0].close();
+    IOUtils.close(indexWriter, taxoWriter);
 
-    IndexTaxonomyReaderPair[] readers = FacetTestUtils.createIndexTaxonomyReaderPair(dirs);
+    DirectoryReader indexReader = DirectoryReader.open(indexDir);
+    TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoDir);
     
     int[] intArray = new int[iParams.getPartitionSize()];
 
     TotalFacetCountsCache tfcc = TotalFacetCountsCache.getSingleton();
     File tmpFile = _TestUtil.createTempFile("test", "tmp", TEMP_DIR);
-    tfcc.store(tmpFile, readers[0].indexReader, readers[0].taxReader, iParams);
+    tfcc.store(tmpFile, indexReader, taxoReader, iParams);
     tfcc.clear(); // not really required because TFCC overrides on load(), but in the test we need not rely on this.
-    tfcc.load(tmpFile, readers[0].indexReader, readers[0].taxReader, iParams);
+    tfcc.load(tmpFile, indexReader, taxoReader, iParams);
     
     // now retrieve the one just loaded
-    TotalFacetCounts totalCounts = tfcc.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams);
+    TotalFacetCounts totalCounts = tfcc.getTotalCounts(indexReader, taxoReader, iParams);
 
     int partition = 0;
-    for (int i=0; i<expectedCounts.length; i+=partitionSize) {
+    for (int i = 0; i < expectedCounts.length; i += partitionSize) {
       totalCounts.fillTotalCountsForPartition(intArray, partition);
       int[] partitionExpectedCounts = new int[partitionSize];
       int nToCopy = Math.min(partitionSize,expectedCounts.length-i);
@@ -103,8 +120,8 @@ public class TestTotalFacetCounts extend
           Arrays.equals(partitionExpectedCounts, intArray));
       ++partition;
     }
-    readers[0].close();
-    IOUtils.close(dirs[0]);
+    IOUtils.close(indexReader, taxoReader);
+    IOUtils.close(indexDir, taxoDir);
     tmpFile.delete();
   }
 

Modified: lucene/dev/branches/lucene4547/lucene/facet/src/test/org/apache/lucene/facet/search/TestTotalFacetCountsCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/facet/src/test/org/apache/lucene/facet/search/TestTotalFacetCountsCache.java?rev=1440839&r1=1440838&r2=1440839&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/facet/src/test/org/apache/lucene/facet/search/TestTotalFacetCountsCache.java (original)
+++ lucene/dev/branches/lucene4547/lucene/facet/src/test/org/apache/lucene/facet/search/TestTotalFacetCountsCache.java Thu Jan 31 03:37:31 2013
@@ -4,21 +4,17 @@ import java.io.File;
 import java.io.IOException;
 import java.util.Collections;
 import java.util.List;
+import java.util.Random;
 
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.analysis.MockTokenizer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.facet.FacetTestCase;
-import org.apache.lucene.facet.FacetTestUtils;
-import org.apache.lucene.facet.FacetTestUtils.IndexTaxonomyReaderPair;
-import org.apache.lucene.facet.FacetTestUtils.IndexTaxonomyWriterPair;
-import org.apache.lucene.facet.example.ExampleResult;
-import org.apache.lucene.facet.example.TestMultiCLExample;
-import org.apache.lucene.facet.example.multiCL.MultiCLIndexer;
-import org.apache.lucene.facet.example.multiCL.MultiCLSearcher;
 import org.apache.lucene.facet.index.FacetFields;
 import org.apache.lucene.facet.index.params.FacetIndexingParams;
 import org.apache.lucene.facet.search.TotalFacetCounts.CreationType;
+import org.apache.lucene.facet.search.params.CountFacetRequest;
+import org.apache.lucene.facet.search.params.FacetSearchParams;
 import org.apache.lucene.facet.search.results.FacetResult;
 import org.apache.lucene.facet.search.results.FacetResultNode;
 import org.apache.lucene.facet.taxonomy.CategoryPath;
@@ -30,6 +26,8 @@ import org.apache.lucene.index.Directory
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.MockDirectoryWrapper;
 import org.apache.lucene.util.IOUtils;
@@ -86,8 +84,8 @@ public class TestTotalFacetCountsCache e
   }
 
   /** Utility method to add a document and facets to an index/taxonomy. */
-  static void addFacets(FacetIndexingParams iParams, IndexWriter iw,
-                        TaxonomyWriter tw, String... strings) throws IOException {
+  private static void addFacets(FacetIndexingParams iParams, IndexWriter iw,
+      TaxonomyWriter tw, String... strings) throws IOException {
     Document doc = new Document();
     FacetFields facetFields = new FacetFields(tw, iParams);
     facetFields.addFields(doc, Collections.singletonList(new CategoryPath(strings)));
@@ -95,7 +93,7 @@ public class TestTotalFacetCountsCache e
   }
 
   /** Clears the cache and sets its size to one. */
-  static void initCache() {
+  private static void initCache() {
     TFC.clear();
     TFC.setCacheSize(1); // Set to keep one in memory
   }
@@ -107,37 +105,35 @@ public class TestTotalFacetCountsCache e
     initCache();
   }
 
-  /** runs a few instances of {@link MultiCLSearcher} in parallel */
+  /** runs few searches in parallel */
   public void testGeneralSynchronization() throws Exception {
-    int numIters = atLeast(2);
+    int numIters = atLeast(4);
+    Random random = random();
     for (int i = 0; i < numIters; i++) {
-      doTestGeneralSynchronization(_TestUtil.nextInt(random(), 2, 4),
-                                  random().nextBoolean() ? -1 : _TestUtil.nextInt(random(), 1, 10),
-                                  _TestUtil.nextInt(random(), 0, 3));
+      int numThreads = random.nextInt(3) + 2; // 2-4
+      int sleepMillis = random.nextBoolean() ? -1 : random.nextInt(10) + 1 /*1-10*/;
+      int cacheSize = random.nextInt(4); // 0-3
+      doTestGeneralSynchronization(numThreads, sleepMillis, cacheSize);
     }
   }
 
-  /**
-   * Run many instances of {@link MultiCLSearcher} in parallel, results should
-   * be sane. Each instance has a random delay for reading bytes, to ensure
-   * that threads finish in different order than started.
-   */
-  @Test @Nightly
-  public void testGeneralSynchronizationBig() throws Exception {
-    int[] numThreads = new int[] { 2, 3, 5, 8 };
-    int[] sleepMillis = new int[] { -1, 1, 20, 33 };
-    int[] cacheSize = new int[] { 0,1,2,3,5 };
-    for (int size : cacheSize) {
-      for (int sleep : sleepMillis) {
-        for (int nThreads : numThreads) {
-          doTestGeneralSynchronization(nThreads, sleep, size);
-        }
-      }
+  private static final String[] CATEGORIES = new String[] { "a/b", "c/d", "a/e", "a/d", "c/g", "c/z", "b/a", "1/2", "b/c" };
+
+  private void index(Directory indexDir, Directory taxoDir) throws IOException {
+    IndexWriter indexWriter = new IndexWriter(indexDir, newIndexWriterConfig(TEST_VERSION_CURRENT, null));
+    TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
+    FacetFields facetFields = new FacetFields(taxoWriter);
+    
+    for (String cat : CATEGORIES) {
+      Document doc = new Document();
+      facetFields.addFields(doc, Collections.singletonList(new CategoryPath(cat, '/')));
+      indexWriter.addDocument(doc);
     }
+    
+    IOUtils.close(indexWriter, taxoWriter);
   }
-
-  private void doTestGeneralSynchronization(int numThreads, int sleepMillis,
-      int cacheSize) throws Exception {
+  
+  private void doTestGeneralSynchronization(int numThreads, int sleepMillis, int cacheSize) throws Exception {
     TFC.setCacheSize(cacheSize);
     SlowRAMDirectory slowIndexDir = new SlowRAMDirectory(-1, random());
     MockDirectoryWrapper indexDir = new MockDirectoryWrapper(random(), slowIndexDir);
@@ -145,7 +141,7 @@ public class TestTotalFacetCountsCache e
     MockDirectoryWrapper taxoDir = new MockDirectoryWrapper(random(), slowTaxoDir);
 
     // Index documents without the "slowness"
-    MultiCLIndexer.index(indexDir, taxoDir);
+    index(indexDir, taxoDir);
 
     slowIndexDir.setSleepMillis(sleepMillis);
     slowTaxoDir.setSleepMillis(sleepMillis);
@@ -161,80 +157,64 @@ public class TestTotalFacetCountsCache e
       private IndexReader indexReader;
       private TaxonomyReader taxoReader;
 
-      public Multi(IndexReader indexReader, TaxonomyReader taxoReader,
-                    FacetIndexingParams iParams) {
+      public Multi(IndexReader indexReader, TaxonomyReader taxoReader, FacetIndexingParams iParams) {
         this.indexReader = indexReader;
         this.taxoReader = taxoReader;
         this.iParams = iParams;
       }
 
-      public ExampleResult getResults() {
-        ExampleResult exampleRes = new ExampleResult();
-        exampleRes.setFacetResults(results);
-        return exampleRes;
+      public List<FacetResult> getResults() {
+        return results;
       }
 
       @Override
       public void run() {
         try {
-          results = MultiCLSearcher.searchWithFacets(indexReader, taxoReader, iParams);
+          FacetSearchParams fsp = new FacetSearchParams(iParams, new CountFacetRequest(new CategoryPath("a"), 10),
+              new CountFacetRequest(new CategoryPath("b"), 10));
+          IndexSearcher searcher = new IndexSearcher(indexReader);
+          FacetsCollector fc = FacetsCollector.create(fsp, indexReader, taxoReader);
+          searcher.search(new MatchAllDocsQuery(), fc);
+          results = fc.getFacetResults();
         } catch (Exception e) {
           throw new RuntimeException(e);
         }
       }
     }
 
-    // Instantiate threads, but do not start them
     Multi[] multis = new Multi[numThreads];
-    for (int i = 0; i < numThreads - 1; i++) {
-      multis[i] = new Multi(slowIndexReader, slowTaxoReader, MultiCLIndexer.MULTI_IPARAMS);
+    for (int i = 0; i < numThreads; i++) {
+      multis[i] = new Multi(slowIndexReader, slowTaxoReader, FacetIndexingParams.ALL_PARENTS);
     }
-    // The last thread uses ONLY the DefaultFacetIndexingParams so that
-    // it references a different TFC cache. This will still result
-    // in valid results, but will only search one of the category lists
-    // instead of all of them.
-    multis[numThreads - 1] = new Multi(slowIndexReader, slowTaxoReader, FacetIndexingParams.ALL_PARENTS);
 
-    // Gentleman, start your engines
     for (Multi m : multis) {
       m.start();
     }
 
     // Wait for threads and get results
-    ExampleResult[] multiResults = new ExampleResult[numThreads];
-    for (int i = 0; i < numThreads; i++) {
-      multis[i].join();
-      multiResults[i] = multis[i].getResults();
-    }
-
-    // Each of the (numThreads-1) should have the same predictable
-    // results, which we test for here.
-    for (int i = 0; i < numThreads - 1; i++) {
-      ExampleResult eResults = multiResults[i];
-      TestMultiCLExample.assertCorrectMultiResults(eResults);
-    }
-
-    // The last thread, which only searched over the
-    // DefaultFacetIndexingParams,
-    // has its own results
-    ExampleResult eResults = multiResults[numThreads - 1];
-    List<FacetResult> results = eResults.getFacetResults();
-    assertEquals(3, results.size());
-    String[] expLabels = new String[] { "5", "5/5", "6/2" };
-    double[] expValues = new double[] { 0.0, 0.0, 1.0 };
-    for (int i = 0; i < 3; i++) {
-      FacetResult result = results.get(i);
-      assertNotNull("Result should not be null", result);
-      FacetResultNode resNode = result.getFacetResultNode();
-      assertEquals("Invalid label", expLabels[i], resNode.label.toString());
-      assertEquals("Invalid value", expValues[i], resNode.value, 0.0);
-      assertEquals("Invalid number of subresults", 0, resNode.subResults.size());
+    String[] expLabelsA = new String[] { "a/d", "a/e", "a/b" };
+    String[] expLabelsB = new String[] { "b/c", "b/a" };
+    for (Multi m : multis) {
+      m.join();
+      List<FacetResult> facetResults = m.getResults();
+      assertEquals("expected two results", 2, facetResults.size());
+      
+      FacetResultNode nodeA = facetResults.get(0).getFacetResultNode();
+      int i = 0;
+      for (FacetResultNode node : nodeA.subResults) {
+        assertEquals("wrong count", 1, (int) node.value);
+        assertEquals(expLabelsA[i++], node.label.toString('/'));
+      }
+      
+      FacetResultNode nodeB = facetResults.get(1).getFacetResultNode();
+      i = 0;
+      for (FacetResultNode node : nodeB.subResults) {
+        assertEquals("wrong count", 1, (int) node.value);
+        assertEquals(expLabelsB[i++], node.label.toString('/'));
+      }
     }
-    // we're done, close the index reader and the taxonomy.
-    slowIndexReader.close();
-    slowTaxoReader.close();
-    indexDir.close();
-    taxoDir.close();
+    
+    IOUtils.close(slowIndexReader, slowTaxoReader, indexDir, taxoDir);
   }
 
   /**
@@ -245,77 +225,78 @@ public class TestTotalFacetCountsCache e
   @Test
   public void testGenerationalConsistency() throws Exception {
     // Create temporary RAMDirectories
-    Directory[][] dirs = FacetTestUtils.createIndexTaxonomyDirs(1);
+    Directory indexDir = newDirectory();
+    Directory taxoDir = newDirectory();
 
     // Create our index/taxonomy writers
-    IndexTaxonomyWriterPair[] writers = FacetTestUtils.createIndexTaxonomyWriterPair(dirs);
+    IndexWriter indexWriter = new IndexWriter(indexDir, newIndexWriterConfig(TEST_VERSION_CURRENT, null));
+    TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
     FacetIndexingParams iParams = FacetIndexingParams.ALL_PARENTS;
 
     // Add a facet to the index
-    addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "a", "b");
+    addFacets(iParams, indexWriter, taxoWriter, "a", "b");
 
     // Commit Changes
-    writers[0].indexWriter.commit();
-    writers[0].taxWriter.commit();
+    indexWriter.commit();
+    taxoWriter.commit();
 
     // Open readers
-    IndexTaxonomyReaderPair[] readers = FacetTestUtils.createIndexTaxonomyReaderPair(dirs);
+    DirectoryReader indexReader = DirectoryReader.open(indexDir);
+    TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoDir);
 
     // As this is the first time we have invoked the TotalFacetCountsManager, 
     // we should expect to compute and not read from disk.
-    TotalFacetCounts totalCounts = 
-      TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams);
+    TotalFacetCounts totalCounts = TFC.getTotalCounts(indexReader, taxoReader, iParams);
     int prevGen = assertRecomputed(totalCounts, 0, "after first attempt to get it!");
 
     // Repeating same operation should pull from the cache - not recomputed. 
     assertTrue("Should be obtained from cache at 2nd attempt",totalCounts == 
-      TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams));
+      TFC.getTotalCounts(indexReader, taxoReader, iParams));
 
     // Repeat the same operation as above. but clear first - now should recompute again
     initCache();
-    totalCounts = TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams);
+    totalCounts = TFC.getTotalCounts(indexReader, taxoReader, iParams);
     prevGen = assertRecomputed(totalCounts, prevGen, "after cache clear, 3rd attempt to get it!");
     
     //store to file
     File outputFile = _TestUtil.createTempFile("test", "tmp", TEMP_DIR);
     initCache();
-    TFC.store(outputFile, readers[0].indexReader, readers[0].taxReader, iParams);
-    totalCounts = TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams);
+    TFC.store(outputFile, indexReader, taxoReader, iParams);
+    totalCounts = TFC.getTotalCounts(indexReader, taxoReader, iParams);
     prevGen = assertRecomputed(totalCounts, prevGen, "after cache clear, 4th attempt to get it!");
 
     //clear and load
     initCache();
-    TFC.load(outputFile, readers[0].indexReader, readers[0].taxReader, iParams);
-    totalCounts = TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams);
+    TFC.load(outputFile, indexReader, taxoReader, iParams);
+    totalCounts = TFC.getTotalCounts(indexReader, taxoReader, iParams);
     prevGen = assertReadFromDisc(totalCounts, prevGen, "after 5th attempt to get it!");
 
     // Add a new facet to the index, commit and refresh readers
-    addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "c", "d");
-    writers[0].indexWriter.close();
-    writers[0].taxWriter.close();
+    addFacets(iParams, indexWriter, taxoWriter, "c", "d");
+    IOUtils.close(indexWriter, taxoWriter);
 
-    DirectoryTaxonomyReader newTaxoReader = TaxonomyReader.openIfChanged(readers[0].taxReader);
+    TaxonomyReader newTaxoReader = TaxonomyReader.openIfChanged(taxoReader);
     assertNotNull(newTaxoReader);
-    assertTrue("should have received more cagtegories in updated taxonomy", newTaxoReader.getSize() > readers[0].taxReader.getSize());
-    readers[0].taxReader.close();
-    readers[0].taxReader = newTaxoReader;
+    assertTrue("should have received more cagtegories in updated taxonomy", newTaxoReader.getSize() > taxoReader.getSize());
+    taxoReader.close();
+    taxoReader = newTaxoReader;
     
-    DirectoryReader r2 = DirectoryReader.openIfChanged(readers[0].indexReader);
+    DirectoryReader r2 = DirectoryReader.openIfChanged(indexReader);
     assertNotNull(r2);
-    readers[0].indexReader.close();
-    readers[0].indexReader = r2;
+    indexReader.close();
+    indexReader = r2;
 
     // now use the new reader - should recompute
-    totalCounts = TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams);
+    totalCounts = TFC.getTotalCounts(indexReader, taxoReader, iParams);
     prevGen = assertRecomputed(totalCounts, prevGen, "after updating the index - 7th attempt!");
 
     // try again - should not recompute
     assertTrue("Should be obtained from cache at 8th attempt",totalCounts == 
-      TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams));
+      TFC.getTotalCounts(indexReader, taxoReader, iParams));
     
-    readers[0].close();
+    IOUtils.close(indexReader, taxoReader);
     outputFile.delete();
-    IOUtils.close(dirs[0]);
+    IOUtils.close(indexDir, taxoDir);
   }
 
   private int assertReadFromDisc(TotalFacetCounts totalCounts, int prevGen, String errMsg) {
@@ -341,10 +322,12 @@ public class TestTotalFacetCountsCache e
    */
   @Test
   public void testGrowingTaxonomy() throws Exception {
-    // Create temporary RAMDirectories
-    Directory[][] dirs = FacetTestUtils.createIndexTaxonomyDirs(1);
+    Directory indexDir = newDirectory();
+    Directory taxoDir = newDirectory();
+    
     // Create our index/taxonomy writers
-    IndexTaxonomyWriterPair[] writers = FacetTestUtils.createIndexTaxonomyWriterPair(dirs);
+    IndexWriter indexWriter = new IndexWriter(indexDir, newIndexWriterConfig(TEST_VERSION_CURRENT, null));
+    TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
     FacetIndexingParams iParams = new FacetIndexingParams() {
       @Override
       public int getPartitionSize() {
@@ -352,37 +335,38 @@ public class TestTotalFacetCountsCache e
       }
     };
     // Add a facet to the index
-    addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "a", "b");
+    addFacets(iParams, indexWriter, taxoWriter, "a", "b");
     // Commit Changes
-    writers[0].indexWriter.commit();
-    writers[0].taxWriter.commit();
+    indexWriter.commit();
+    taxoWriter.commit();
 
-    IndexTaxonomyReaderPair[] readers = FacetTestUtils.createIndexTaxonomyReaderPair(dirs);
+    DirectoryReader indexReader = DirectoryReader.open(indexDir);
+    TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoDir);
 
     // Create TFC and write cache to disk
     File outputFile = _TestUtil.createTempFile("test", "tmp", TEMP_DIR);
-    TFC.store(outputFile, readers[0].indexReader, readers[0].taxReader, iParams);
+    TFC.store(outputFile, indexReader, taxoReader, iParams);
     
     // Make the taxonomy grow without touching the index
     for (int i = 0; i < 10; i++) {
-      writers[0].taxWriter.addCategory(new CategoryPath("foo", Integer.toString(i)));
+      taxoWriter.addCategory(new CategoryPath("foo", Integer.toString(i)));
     }
-    writers[0].taxWriter.commit();
-    DirectoryTaxonomyReader newTaxoReader = TaxonomyReader.openIfChanged(readers[0].taxReader);
+    taxoWriter.commit();
+    TaxonomyReader newTaxoReader = TaxonomyReader.openIfChanged(taxoReader);
     assertNotNull(newTaxoReader);
-    readers[0].taxReader.close();
-    readers[0].taxReader = newTaxoReader;
+    taxoReader.close();
+    taxoReader = newTaxoReader;
 
     initCache();
 
     // With the bug, this next call should result in an exception
-    TFC.load(outputFile, readers[0].indexReader, readers[0].taxReader, iParams);
-    TotalFacetCounts totalCounts = TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams);
+    TFC.load(outputFile, indexReader, taxoReader, iParams);
+    TotalFacetCounts totalCounts = TFC.getTotalCounts(indexReader, taxoReader, iParams);
     assertReadFromDisc(totalCounts, 0, "after reading from disk.");
+    
     outputFile.delete();
-    writers[0].close();
-    readers[0].close();
-    IOUtils.close(dirs[0]);
+    IOUtils.close(indexWriter, taxoWriter, indexReader, taxoReader);
+    IOUtils.close(indexDir, taxoDir);
   }
 
   /**
@@ -445,46 +429,52 @@ public class TestTotalFacetCountsCache e
    */
   @Test
   public void testMultipleIndices() throws IOException {
-    // Create temporary RAMDirectories
-    Directory[][] dirs = FacetTestUtils.createIndexTaxonomyDirs(2);
+    Directory indexDir1 = newDirectory(), indexDir2 = newDirectory();
+    Directory taxoDir1 = newDirectory(), taxoDir2 = newDirectory();
+    
     // Create our index/taxonomy writers
-    IndexTaxonomyWriterPair[] writers = FacetTestUtils.createIndexTaxonomyWriterPair(dirs);
+    IndexWriter indexWriter1 = new IndexWriter(indexDir1, newIndexWriterConfig(TEST_VERSION_CURRENT, null));
+    IndexWriter indexWriter2 = new IndexWriter(indexDir2, newIndexWriterConfig(TEST_VERSION_CURRENT, null));
+    TaxonomyWriter taxoWriter1 = new DirectoryTaxonomyWriter(taxoDir1);
+    TaxonomyWriter taxoWriter2 = new DirectoryTaxonomyWriter(taxoDir2);
     FacetIndexingParams iParams = FacetIndexingParams.ALL_PARENTS;
 
     // Add a facet to the index
-    addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "a", "b");
-    addFacets(iParams, writers[1].indexWriter, writers[1].taxWriter, "d", "e");
+    addFacets(iParams, indexWriter1, taxoWriter1, "a", "b");
+    addFacets(iParams, indexWriter1, taxoWriter1, "d", "e");
     // Commit Changes
-    writers[0].indexWriter.commit();
-    writers[0].taxWriter.commit();
-    writers[1].indexWriter.commit();
-    writers[1].taxWriter.commit();
+    indexWriter1.commit();
+    indexWriter2.commit();
+    taxoWriter1.commit();
+    taxoWriter2.commit();
 
     // Open two readers
-    IndexTaxonomyReaderPair[] readers = FacetTestUtils.createIndexTaxonomyReaderPair(dirs);
+    DirectoryReader indexReader1 = DirectoryReader.open(indexDir1);
+    DirectoryReader indexReader2 = DirectoryReader.open(indexDir2);
+    TaxonomyReader taxoReader1 = new DirectoryTaxonomyReader(taxoDir1);
+    TaxonomyReader taxoReader2 = new DirectoryTaxonomyReader(taxoDir2);
 
     // As this is the first time we have invoked the TotalFacetCountsManager, we
     // should expect to compute.
-    TotalFacetCounts totalCounts0 = 
-      TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams);
+    TotalFacetCounts totalCounts0 = TFC.getTotalCounts(indexReader1, taxoReader1, iParams);
     int prevGen = -1;
     prevGen = assertRecomputed(totalCounts0, prevGen, "after attempt 1");
     assertTrue("attempt 1b for same input [0] shout find it in cache",
-        totalCounts0 == TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams));
+        totalCounts0 == TFC.getTotalCounts(indexReader1, taxoReader1, iParams));
     
     // 2nd Reader - As this is the first time we have invoked the
     // TotalFacetCountsManager, we should expect a state of NEW to be returned.
-    TotalFacetCounts totalCounts1 = TFC.getTotalCounts(readers[1].indexReader, readers[1].taxReader, iParams);
+    TotalFacetCounts totalCounts1 = TFC.getTotalCounts(indexReader2, taxoReader2, iParams);
     prevGen = assertRecomputed(totalCounts1, prevGen, "after attempt 2");
     assertTrue("attempt 2b for same input [1] shout find it in cache",
-        totalCounts1 == TFC.getTotalCounts(readers[1].indexReader, readers[1].taxReader, iParams));
+        totalCounts1 == TFC.getTotalCounts(indexReader2, taxoReader2, iParams));
 
     // Right now cache size is one, so first TFC is gone and should be recomputed  
-    totalCounts0 = TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams);
+    totalCounts0 = TFC.getTotalCounts(indexReader1, taxoReader1, iParams);
     prevGen = assertRecomputed(totalCounts0, prevGen, "after attempt 3");
     
     // Similarly will recompute the second result  
-    totalCounts1 = TFC.getTotalCounts(readers[1].indexReader, readers[1].taxReader, iParams);
+    totalCounts1 = TFC.getTotalCounts(indexReader2, taxoReader2, iParams);
     prevGen = assertRecomputed(totalCounts1, prevGen, "after attempt 4");
 
     // Now we set the cache size to two, meaning both should exist in the
@@ -492,23 +482,19 @@ public class TestTotalFacetCountsCache e
     TFC.setCacheSize(2);
 
     // Re-compute totalCounts0 (was evicted from the cache when the cache was smaller)
-    totalCounts0 = TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams);
+    totalCounts0 = TFC.getTotalCounts(indexReader1, taxoReader1, iParams);
     prevGen = assertRecomputed(totalCounts0, prevGen, "after attempt 5");
 
     // now both are in the larger cache and should not be recomputed 
-    totalCounts1 = TFC.getTotalCounts(readers[1].indexReader, readers[1].taxReader, iParams);
+    totalCounts1 = TFC.getTotalCounts(indexReader2, taxoReader2, iParams);
     assertTrue("with cache of size 2 res no. 0 should come from cache",
-        totalCounts0 == TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams));
+        totalCounts0 == TFC.getTotalCounts(indexReader1, taxoReader1, iParams));
     assertTrue("with cache of size 2 res no. 1 should come from cache",
-        totalCounts1 == TFC.getTotalCounts(readers[1].indexReader, readers[1].taxReader, iParams));
+        totalCounts1 == TFC.getTotalCounts(indexReader2, taxoReader2, iParams));
     
-    writers[0].close();
-    writers[1].close();
-    readers[0].close();
-    readers[1].close();
-    for (Directory[] dirset : dirs) {
-      IOUtils.close(dirset);
-    }
+    IOUtils.close(indexWriter1, indexWriter2, taxoWriter1, taxoWriter2);
+    IOUtils.close(indexReader1, indexReader2, taxoReader1, taxoReader2);
+    IOUtils.close(indexDir1, indexDir2, taxoDir1, taxoDir2);
   }
 
 }

Modified: lucene/dev/branches/lucene4547/lucene/facet/src/test/org/apache/lucene/facet/search/sampling/BaseSampleTestTopK.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/facet/src/test/org/apache/lucene/facet/search/sampling/BaseSampleTestTopK.java?rev=1440839&r1=1440838&r2=1440839&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/facet/src/test/org/apache/lucene/facet/search/sampling/BaseSampleTestTopK.java (original)
+++ lucene/dev/branches/lucene4547/lucene/facet/src/test/org/apache/lucene/facet/search/sampling/BaseSampleTestTopK.java Thu Jan 31 03:37:31 2013
@@ -3,6 +3,7 @@ package org.apache.lucene.facet.search.s
 import java.util.List;
 import java.util.Random;
 
+import org.apache.lucene.facet.index.params.FacetIndexingParams;
 import org.apache.lucene.facet.search.BaseTestTopK;
 import org.apache.lucene.facet.search.FacetsAccumulator;
 import org.apache.lucene.facet.search.FacetsCollector;
@@ -46,8 +47,8 @@ public abstract class BaseSampleTestTopK
   protected static final int RETRIES = 10;
   
   @Override
-  protected FacetSearchParams searchParamsWithRequests(int numResults, int partitionSize) {
-    FacetSearchParams res = super.searchParamsWithRequests(numResults, partitionSize);
+  protected FacetSearchParams searchParamsWithRequests(int numResults, FacetIndexingParams fip) {
+    FacetSearchParams res = super.searchParamsWithRequests(numResults, fip);
     for (FacetRequest req : res.facetRequests) {
       // randomize the way we aggregate results
       if (random().nextBoolean()) {
@@ -71,20 +72,23 @@ public abstract class BaseSampleTestTopK
     boolean useRandomSampler = random().nextBoolean();
     for (int partitionSize : partitionSizes) {
       try {
-        initIndex(partitionSize);
+        // complements return counts for all ordinals, so force ALL_PARENTS indexing
+        // so that it's easier to compare
+        FacetIndexingParams fip = getFacetIndexingParams(partitionSize, true);
+        initIndex(fip);
         // Get all of the documents and run the query, then do different
         // facet counts and compare to control
         Query q = new TermQuery(new Term(CONTENT_FIELD, BETA)); // 90% of the docs
         ScoredDocIdCollector docCollector = ScoredDocIdCollector.create(indexReader.maxDoc(), false);
         
-        FacetSearchParams expectedSearchParams = searchParamsWithRequests(K, partitionSize); 
+        FacetSearchParams expectedSearchParams = searchParamsWithRequests(K, fip); 
         FacetsCollector fc = FacetsCollector.create(expectedSearchParams, indexReader, taxoReader);
         
         searcher.search(q, MultiCollector.wrap(docCollector, fc));
         
         List<FacetResult> expectedResults = fc.getFacetResults();
         
-        FacetSearchParams samplingSearchParams = searchParamsWithRequests(K, partitionSize); 
+        FacetSearchParams samplingSearchParams = searchParamsWithRequests(K, fip); 
         
         // try several times in case of failure, because the test has a chance to fail 
         // if the top K facets are not sufficiently common with the sample set

Modified: lucene/dev/branches/lucene4547/lucene/facet/src/test/org/apache/lucene/facet/search/sampling/OversampleWithDepthTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/facet/src/test/org/apache/lucene/facet/search/sampling/OversampleWithDepthTest.java?rev=1440839&r1=1440838&r2=1440839&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/facet/src/test/org/apache/lucene/facet/search/sampling/OversampleWithDepthTest.java (original)
+++ lucene/dev/branches/lucene4547/lucene/facet/src/test/org/apache/lucene/facet/search/sampling/OversampleWithDepthTest.java Thu Jan 31 03:37:31 2013
@@ -3,7 +3,6 @@ package org.apache.lucene.facet.search.s
 import java.io.IOException;
 import java.util.Collections;
 
-import org.apache.lucene.analysis.core.KeywordAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.facet.FacetTestCase;
 import org.apache.lucene.facet.index.FacetFields;
@@ -95,7 +94,7 @@ public class OversampleWithDepthTest ext
   }
 
   private void index100Docs(Directory indexDir, Directory taxoDir, FacetIndexingParams fip) throws IOException {
-    IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new KeywordAnalyzer());
+    IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, null);
     IndexWriter w = new IndexWriter(indexDir, iwc);
     TaxonomyWriter tw = new DirectoryTaxonomyWriter(taxoDir);
     

Modified: lucene/dev/branches/lucene4547/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyReader.java?rev=1440839&r1=1440838&r2=1440839&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyReader.java (original)
+++ lucene/dev/branches/lucene4547/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyReader.java Thu Jan 31 03:37:31 2013
@@ -3,7 +3,7 @@ package org.apache.lucene.facet.taxonomy
 import java.io.IOException;
 import java.util.Random;
 
-import org.apache.lucene.analysis.core.KeywordAnalyzer;
+import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.facet.FacetTestCase;
 import org.apache.lucene.facet.taxonomy.CategoryPath;
 import org.apache.lucene.facet.taxonomy.TaxonomyReader;
@@ -256,7 +256,7 @@ public class TestDirectoryTaxonomyReader
     // hold onto IW to forceMerge
     // note how we don't close it, since DTW will close it.
     final IndexWriter iw = new IndexWriter(dir,
-        new IndexWriterConfig(TEST_VERSION_CURRENT, new KeywordAnalyzer())
+        new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))
             .setMergePolicy(new LogByteSizeMergePolicy()));
     DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir) {
       @Override
@@ -299,7 +299,7 @@ public class TestDirectoryTaxonomyReader
     // hold onto IW to forceMerge
     // note how we don't close it, since DTW will close it.
     final IndexWriter iw = new IndexWriter(dir,
-        new IndexWriterConfig(TEST_VERSION_CURRENT, new KeywordAnalyzer())
+        new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))
             .setMergePolicy(new LogByteSizeMergePolicy()));
     DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir) {
       @Override

Modified: lucene/dev/branches/lucene4547/lucene/site/xsl/index.xsl
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/site/xsl/index.xsl?rev=1440839&r1=1440838&r2=1440839&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/site/xsl/index.xsl (original)
+++ lucene/dev/branches/lucene4547/lucene/site/xsl/index.xsl Thu Jan 31 03:37:31 2013
@@ -58,7 +58,8 @@
         <li><a href="core/overview-summary.html#overview_description">Introduction to Lucene's APIs</a>:
         High-level summary of the different Lucene packages. </li>
         <li><a href="core/org/apache/lucene/analysis/package-summary.html#package_description">Analysis overview</a>:
-        Introduction to Lucene's analysis API. </li>
+        Introduction to Lucene's analysis API.  See also the
+        <a href="core/org/apache/lucene/analysis/TokenStream.html">TokenStream consumer workflow</a>.</li>
         </ul>
         <h2>Reference Documents</h2>
           <ul>

Modified: lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/analysis/MockCharFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/analysis/MockCharFilter.java?rev=1440839&r1=1440838&r2=1440839&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/analysis/MockCharFilter.java (original)
+++ lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/analysis/MockCharFilter.java Thu Jan 31 03:37:31 2013
@@ -19,6 +19,7 @@ package org.apache.lucene.analysis;
 
 import java.io.IOException;
 import java.io.Reader;
+import java.util.Map;
 import java.util.SortedMap;
 import java.util.TreeMap;
 
@@ -89,8 +90,8 @@ public class MockCharFilter extends Char
 
   @Override
   public int correct(int currentOff) {
-    SortedMap<Integer,Integer> subMap = corrections.subMap(0, currentOff+1);
-    int ret = subMap.isEmpty() ? currentOff : currentOff + subMap.get(subMap.lastKey());
+    Map.Entry<Integer,Integer> lastEntry = corrections.lowerEntry(currentOff+1);
+    int ret = lastEntry == null ? currentOff : currentOff + lastEntry.getValue();
     assert ret >= 0 : "currentOff=" + currentOff + ",diff=" + (ret-currentOff);
     return ret;
   }

Modified: lucene/dev/branches/lucene4547/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/CHANGES.txt?rev=1440839&r1=1440838&r2=1440839&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/CHANGES.txt (original)
+++ lucene/dev/branches/lucene4547/solr/CHANGES.txt Thu Jan 31 03:37:31 2013
@@ -95,6 +95,11 @@ Bug Fixes
 
 * SOLR-4342: Fix DataImportHandler stats to be a prper Map (hossman)
 
+* SOLR-3967: langid.enforceSchema option checks source field instead of target field (janhoy)
+
+* SOLR-4380: Replicate after startup option would not replicate until the
+  IndexWriter was lazily opened. (Mark Miller, Gregg Donovan)
+
 Optimizations
 ----------------------
 
@@ -115,6 +120,8 @@ Optimizations
 * SOLR-4284: Admin UI - make core list scrollable separate from the rest of
   the UI (steffkes)
 
+* SOLR-4364: Admin UI - Locale based number formatting (steffkes)
+
 Other Changes
 ----------------------
 
@@ -123,6 +130,8 @@ Other Changes
 
 * SOLR-4353: Renamed example jetty context file to reduce confusion (hossman)
 
+* SOLR-4384: Make post.jar report timing information (Upayavira via janhoy)
+
 ==================  4.1.0 ==================
 
 Versions of Major Components

Modified: lucene/dev/branches/lucene4547/solr/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/build.xml?rev=1440839&r1=1440838&r2=1440839&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/build.xml (original)
+++ lucene/dev/branches/lucene4547/solr/build.xml Thu Jan 31 03:37:31 2013
@@ -560,17 +560,8 @@
     <echo message="Checking for broken links..."/>
     <check-broken-links dir="${javadoc.dir}"/>
     <echo message="Checking for malformed docs..."/>
-    <!-- TODO: add missing package.htmls and bump this to level=package -->
-    <check-missing-javadocs dir="${javadoc.dir}" level="none"/>
-    <!-- prevent the modules without problems from getting worse -->
-    <check-missing-javadocs dir="${javadoc.dir}/solr-analysis-extras" level="package"/>
-    <check-missing-javadocs dir="${javadoc.dir}/solr-cell" level="package"/>
-    <check-missing-javadocs dir="${javadoc.dir}/solr-clustering" level="package"/>
-    <!-- solr-core: problems: -->
-    <check-missing-javadocs dir="${javadoc.dir}/solr-dataimporthandler" level="package"/>
-    <check-missing-javadocs dir="${javadoc.dir}/solr-dataimporthandler-extras" level="package"/>
-    <check-missing-javadocs dir="${javadoc.dir}/solr-solrj" level="package"/>
-    <check-missing-javadocs dir="${javadoc.dir}/solr-test-framework" level="package"/>
+    <!-- TODO: add missing docs for all classes and bump this to level=class -->
+    <check-missing-javadocs dir="${javadoc.dir}" level="package"/>
   </target>
  
   <target name="-ecj-javadoc-lint" depends="documentation,compile-solr-test-framework,-ecj-resolve">

Modified: lucene/dev/branches/lucene4547/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/schema.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/schema.xml?rev=1440839&r1=1440838&r2=1440839&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/schema.xml (original)
+++ lucene/dev/branches/lucene4547/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/schema.xml Thu Jan 31 03:37:31 2013
@@ -23,9 +23,6 @@
      kitchen sink thrown in. See example/solr/conf/schema.xml for a 
      more concise example.
 
-     $Id: schema.xml 382610 2006-03-03 01:43:03Z yonik $
-     $Source: /cvs/main/searching/solr-configs/test/WEB-INF/classes/schema.xml,v $
-     $Name:  $
   -->
 
 <schema name="test" version="1.0">

Modified: lucene/dev/branches/lucene4547/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/solrconfig.xml?rev=1440839&r1=1440838&r2=1440839&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/solrconfig.xml (original)
+++ lucene/dev/branches/lucene4547/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/solrconfig.xml Thu Jan 31 03:37:31 2013
@@ -17,11 +17,6 @@
  limitations under the License.
 -->
 
-<!-- $Id: solrconfig.xml 382610 2006-03-03 01:43:03Z yonik $
-     $Source$
-     $Name$
-  -->
-
 <config>
   <luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
   <jmx />

Modified: lucene/dev/branches/lucene4547/solr/contrib/langid/src/java/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/contrib/langid/src/java/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessor.java?rev=1440839&r1=1440838&r2=1440839&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/contrib/langid/src/java/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessor.java (original)
+++ lucene/dev/branches/lucene4547/solr/contrib/langid/src/java/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessor.java Thu Jan 31 03:37:31 2013
@@ -222,10 +222,6 @@ public abstract class LanguageIdentifier
             log.debug("Mapping field "+fieldName+" using document global language "+fieldLang);
           }
           String mappedOutputField = getMappedField(fieldName, fieldLang);
-          if(enforceSchema && schema.getFieldOrNull(fieldName) == null) {
-            log.warn("Unsuccessful field name mapping to {}, field does not exist, skipping mapping.", mappedOutputField, fieldName);
-            mappedOutputField = fieldName;
-          }
 
           if (mappedOutputField != null) {
             log.debug("Mapping field {} to {}", doc.getFieldValue(docIdField), fieldLang);
@@ -350,17 +346,23 @@ public abstract class LanguageIdentifier
 
   /**
    * Returns the name of the field to map the current contents into, so that they are properly analyzed.  For instance
-   * if the currentField is "text" and the code is "en", the new field would be "text_en".  If such a field doesn't exist,
-   * then null is returned.
+   * if the currentField is "text" and the code is "en", the new field would by default be "text_en".
+   * This method also performs custom regex pattern replace if configured. If enforceSchema=true
+   * and the resulting field name doesn't exist, then null is returned.
    *
    * @param currentField The current field name
    * @param language the language code
-   * @return The new schema field name, based on pattern and replace
+   * @return The new schema field name, based on pattern and replace, or null if illegal
    */
   protected String getMappedField(String currentField, String language) {
     String lc = lcMap.containsKey(language) ? lcMap.get(language) : language;
     String newFieldName = langPattern.matcher(mapPattern.matcher(currentField).replaceFirst(mapReplaceStr)).replaceFirst(lc);
-    log.debug("Doing mapping from "+currentField+" with language "+language+" to field "+newFieldName);
+    if(enforceSchema && schema.getFieldOrNull(newFieldName) == null) {
+      log.warn("Unsuccessful field name mapping from {} to {}, field does not exist and enforceSchema=true; skipping mapping.", currentField, newFieldName);
+      return null;
+    } else {
+      log.debug("Doing mapping from "+currentField+" with language "+language+" to field "+newFieldName);
+    }
     return newFieldName;
   }
 

Modified: lucene/dev/branches/lucene4547/solr/contrib/langid/src/test-files/langid/solr/collection1/conf/schema.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/contrib/langid/src/test-files/langid/solr/collection1/conf/schema.xml?rev=1440839&r1=1440838&r2=1440839&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/contrib/langid/src/test-files/langid/solr/collection1/conf/schema.xml (original)
+++ lucene/dev/branches/lucene4547/solr/contrib/langid/src/test-files/langid/solr/collection1/conf/schema.xml Thu Jan 31 03:37:31 2013
@@ -23,9 +23,6 @@
      kitchen sink thrown in. See example/solr/conf/schema.xml for a
      more concise example.
 
-     $Id: schema.xml 382610 2006-03-03 01:43:03Z yonik $
-     $Source: /cvs/main/searching/solr-configs/test/WEB-INF/classes/schema.xml,v $
-     $Name:  $
   -->
 
 <schema name="test" version="1.5">

Modified: lucene/dev/branches/lucene4547/solr/contrib/langid/src/test/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessorFactoryTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/contrib/langid/src/test/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessorFactoryTestCase.java?rev=1440839&r1=1440838&r2=1440839&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/contrib/langid/src/test/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessorFactoryTestCase.java (original)
+++ lucene/dev/branches/lucene4547/solr/contrib/langid/src/test/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessorFactoryTestCase.java Thu Jan 31 03:37:31 2013
@@ -93,7 +93,7 @@ public abstract class LanguageIdentifier
     parameters = new ModifiableSolrParams();
     parameters.add("langid.fl", "name");
     parameters.add("langid.map.lcmap", "jp:s zh:cjk ko:cjk");
-    parameters.add("langid.enforceSchema", "true");
+    parameters.set("langid.enforceSchema", "false");
     liProcessor = createLangIdProcessor(parameters);
     
     assertEquals("test_no", liProcessor.getMappedField("test", "no"));
@@ -102,13 +102,17 @@ public abstract class LanguageIdentifier
     assertEquals("test_cjk", liProcessor.getMappedField("test", "zh"));
     assertEquals("test_cjk", liProcessor.getMappedField("test", "ko"));
 
-    // Prove support for other mapping regex
-    parameters.add("langid.map.pattern", "text_(.*?)_field");
-    parameters.add("langid.map.replace", "$1_{lang}Text");
+    // Test that enforceSchema correctly catches illegal field and returns null
+    parameters.set("langid.enforceSchema", "true");
     liProcessor = createLangIdProcessor(parameters);
+    assertEquals(null, liProcessor.getMappedField("inputfield", "sv"));
 
-    assertEquals("title_noText", liProcessor.getMappedField("text_title_field", "no"));
-    assertEquals("body_svText", liProcessor.getMappedField("text_body_field", "sv"));
+    // Prove support for other mapping regex, still with enforceSchema=true
+    parameters.add("langid.map.pattern", "text_(.*?)_field");
+    parameters.add("langid.map.replace", "$1_{lang}_s");
+    liProcessor = createLangIdProcessor(parameters);
+    assertEquals("title_no_s", liProcessor.getMappedField("text_title_field", "no"));
+    assertEquals("body_sv_s", liProcessor.getMappedField("text_body_field", "sv"));
   }
 
   @Test

Modified: lucene/dev/branches/lucene4547/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java?rev=1440839&r1=1440838&r2=1440839&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java Thu Jan 31 03:37:31 2013
@@ -42,6 +42,7 @@ import org.apache.commons.io.IOUtils;
 import org.apache.lucene.index.IndexCommit;
 import org.apache.lucene.index.IndexDeletionPolicy;
 import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
@@ -877,9 +878,9 @@ public class ReplicationHandler extends 
             }
           }
 
-          // reboot the writer on the new index
-          // TODO: perhaps this is no longer necessary then?
-         // core.getUpdateHandler().newIndexWriter(true);
+          // ensure the writer is init'd so that we have a list of commit points
+          RefCounted<IndexWriter> iw = core.getUpdateHandler().getSolrCoreState().getIndexWriter(core);
+          iw.decref();
 
         } catch (IOException e) {
           LOG.warn("Unable to get IndexCommit on startup", e);

Modified: lucene/dev/branches/lucene4547/solr/core/src/java/org/apache/solr/util/SimplePostTool.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/java/org/apache/solr/util/SimplePostTool.java?rev=1440839&r1=1440838&r2=1440839&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/java/org/apache/solr/util/SimplePostTool.java (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/java/org/apache/solr/util/SimplePostTool.java Thu Jan 31 03:37:31 2013
@@ -165,6 +165,7 @@ public class SimplePostTool {
    * This method delegates to the correct mode method.
    */
   public void execute() {
+    final long startTime = System.currentTimeMillis();
     if (DATA_MODE_FILES.equals(mode) && args.length > 0) {
       doFilesMode();
     } else if(DATA_MODE_ARGS.equals(mode) && args.length > 0) {
@@ -180,9 +181,31 @@ public class SimplePostTool {
     
     if (commit)   commit();
     if (optimize) optimize();
+    final long endTime = System.currentTimeMillis();
+    displayTiming(endTime - startTime);
   }
   
   /**
+   * Pretty prints the number of milliseconds taken to post the content to Solr
+   * @param millis the time in milliseconds
+   */
+  private void displayTiming(long millis) {
+    // TODO: if the intent is user-display: this should use SimpleDateFormat 
+    // or similar instead of homemade formatting.
+    long hours = millis / 3600000;
+    long minutes = (millis / 60000) % 60;
+    long seconds = (millis / 1000) % 60;
+    long milliseconds = millis % 1000;
+    if (hours>0) {
+      System.out.println(String.format(Locale.getDefault(), "Time taken: %02d:%02d:%02d.%03d", hours, minutes, seconds, milliseconds));
+    } else if (minutes>0) {
+      System.out.println(String.format(Locale.getDefault(), "Time taken: %02d:%02d.%03d", minutes, seconds, milliseconds));
+    } else {
+      System.out.println(String.format(Locale.getDefault(), "Time taken: %d.%03ds", seconds, milliseconds));
+    }
+ }
+
+  /**
    * Parses incoming arguments and system params and initializes the tool
    * @param args the incoming cmd line args
    * @return an instance of SimplePostTool

Modified: lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/bad_solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/bad_solrconfig.xml?rev=1440839&r1=1440838&r2=1440839&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/bad_solrconfig.xml (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/bad_solrconfig.xml Thu Jan 31 03:37:31 2013
@@ -17,10 +17,6 @@
  limitations under the License.
 -->
 
-<!-- $Id: solrconfig.xml 382610 2006-03-03 01:43:03Z yonik $
-     $Source$
-     $Name$
-  -->
 
 <config>
   <luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>

Modified: lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/schema-binaryfield.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/schema-binaryfield.xml?rev=1440839&r1=1440838&r2=1440839&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/schema-binaryfield.xml (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/schema-binaryfield.xml Thu Jan 31 03:37:31 2013
@@ -23,9 +23,6 @@
      kitchen sink thrown in. See example/solr/conf/schema.xml for a
      more concise example.
 
-     $Id: schema-copyfield-test.xml 721758 2008-11-30 04:46:20Z koji $
-     $Source: /cvs/main/searching/solr-configs/test/WEB-INF/classes/schema.xml,v $
-     $Name:  $
   -->
 
 <schema name="test" version="1.2">

Modified: lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/schema-copyfield-test.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/schema-copyfield-test.xml?rev=1440839&r1=1440838&r2=1440839&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/schema-copyfield-test.xml (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/schema-copyfield-test.xml Thu Jan 31 03:37:31 2013
@@ -23,9 +23,6 @@
      kitchen sink thrown in. See example/solr/conf/schema.xml for a 
      more concise example.
 
-     $Id$
-     $Source: /cvs/main/searching/solr-configs/test/WEB-INF/classes/schema.xml,v $
-     $Name:  $
   -->
 
 <schema name="test" version="1.0">

Modified: lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/schema-not-required-unique-key.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/schema-not-required-unique-key.xml?rev=1440839&r1=1440838&r2=1440839&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/schema-not-required-unique-key.xml (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/schema-not-required-unique-key.xml Thu Jan 31 03:37:31 2013
@@ -20,8 +20,6 @@
     Striped down schema used to make sure an explicit required=false is
     observed for the uniqueKey field
 
-    $Id$
-    $Source: /cvs/main/searching/solr-configs/test/WEB-INF/classes/schema.xml,v $
   -->
 <schema name="test" version="1.0">
   <types>

Modified: lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/schema-numeric.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/schema-numeric.xml?rev=1440839&r1=1440838&r2=1440839&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/schema-numeric.xml (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/schema-numeric.xml Thu Jan 31 03:37:31 2013
@@ -23,9 +23,6 @@
      kitchen sink thrown in. See example/solr/conf/schema.xml for a 
      more concise example.
 
-     $Id: schema.xml 382610 2006-03-03 01:43:03Z yonik $
-     $Source: /cvs/main/searching/solr-configs/test/WEB-INF/classes/schema.xml,v $
-     $Name:  $
   -->
 
 <schema name="test" version="1.0">

Modified: lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/schema-replication1.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/schema-replication1.xml?rev=1440839&r1=1440838&r2=1440839&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/schema-replication1.xml (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/schema-replication1.xml Thu Jan 31 03:37:31 2013
@@ -23,9 +23,6 @@
      kitchen sink thrown in. See example/solr/conf/schema.xml for a 
      more concise example.
 
-     $Id$
-     $Source$
-     $Name$
   -->
 
 <schema name="test" version="1.2">

Modified: lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/schema-replication2.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/schema-replication2.xml?rev=1440839&r1=1440838&r2=1440839&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/schema-replication2.xml (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/schema-replication2.xml Thu Jan 31 03:37:31 2013
@@ -23,9 +23,6 @@
      kitchen sink thrown in. See example/solr/conf/schema.xml for a 
      more concise example.
 
-     $Id$
-     $Source$
-     $Name$
   -->
 
 <schema name="test" version="1.2">

Modified: lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/schema-required-fields.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/schema-required-fields.xml?rev=1440839&r1=1440838&r2=1440839&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/schema-required-fields.xml (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/schema-required-fields.xml Thu Jan 31 03:37:31 2013
@@ -23,9 +23,6 @@
      kitchen sink thrown in. See example/solr/conf/schema.xml for a 
      more concise example.
 
-     $Id$
-     $Source: /cvs/main/searching/solr-configs/test/WEB-INF/classes/schema.xml,v $
-     $Name:  $
   -->
 
 <schema name="test" version="1.0">

Modified: lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/schema-stop-keep.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/schema-stop-keep.xml?rev=1440839&r1=1440838&r2=1440839&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/schema-stop-keep.xml (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/schema-stop-keep.xml Thu Jan 31 03:37:31 2013
@@ -20,9 +20,6 @@
 
   For testing stopword configuration and keep word configuration
 
-     $Id: schema.xml 382610 2006-03-03 01:43:03Z yonik $
-     $Source: /cvs/main/searching/solr-configs/test/WEB-INF/classes/schema.xml,v $
-     $Name:  $
   -->
 
 <schema name="test" version="1.0">

Modified: lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/schema.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/schema.xml?rev=1440839&r1=1440838&r2=1440839&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/schema.xml (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/schema.xml Thu Jan 31 03:37:31 2013
@@ -23,9 +23,6 @@
      kitchen sink thrown in. See example/solr/conf/schema.xml for a
      more concise example.
 
-     $Id: schema.xml 382610 2006-03-03 01:43:03Z yonik $
-     $Source: /cvs/main/searching/solr-configs/test/WEB-INF/classes/schema.xml,v $
-     $Name:  $
   -->
 
 <schema name="test" version="1.0">

Modified: lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/schema12.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/schema12.xml?rev=1440839&r1=1440838&r2=1440839&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/schema12.xml (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/schema12.xml Thu Jan 31 03:37:31 2013
@@ -23,9 +23,6 @@
      kitchen sink thrown in. See example/solr/conf/schema.xml for a 
      more concise example.
 
-     $Id$
-     $Source: /cvs/main/searching/solr-configs/test/WEB-INF/classes/schema.xml,v $
-     $Name:  $
   -->
 
 <schema name="test" version="1.4">

Modified: lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/schemasurround.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/schemasurround.xml?rev=1440839&r1=1440838&r2=1440839&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/schemasurround.xml (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/schemasurround.xml Thu Jan 31 03:37:31 2013
@@ -23,9 +23,6 @@
      kitchen sink thrown in. See example/solr/conf/schema.xml for a 
      more concise example.
 
-     $Id: schema12.xml 1149050 2011-07-21 07:09:27Z koji $
-     $Source: /cvs/main/searching/solr-configs/test/WEB-INF/classes/schema.xml,v $
-     $Name:  $
   -->
 
 <schema name="test" version="1.4">

Modified: lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/solrconfig-SOLR-749.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/solrconfig-SOLR-749.xml?rev=1440839&r1=1440838&r2=1440839&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/solrconfig-SOLR-749.xml (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/solrconfig-SOLR-749.xml Thu Jan 31 03:37:31 2013
@@ -17,11 +17,6 @@
  limitations under the License.
 -->
 
-<!-- $Id: solrconfig.xml 382610 2006-03-03 01:43:03Z yonik $
-     $Source$
-     $Name$
-  -->
-
 <config>
   <luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
   <directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.RAMDirectoryFactory}"/>

Modified: lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/solrconfig-delpolicy1.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/solrconfig-delpolicy1.xml?rev=1440839&r1=1440838&r2=1440839&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/solrconfig-delpolicy1.xml (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/solrconfig-delpolicy1.xml Thu Jan 31 03:37:31 2013
@@ -17,10 +17,6 @@
  limitations under the License.
 -->
 
-<!-- $Id$
-     $Source$
-     $Name$
-  -->
 
 <config>
   <luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>

Modified: lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/solrconfig-elevate.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/solrconfig-elevate.xml?rev=1440839&r1=1440838&r2=1440839&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/solrconfig-elevate.xml (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/solrconfig-elevate.xml Thu Jan 31 03:37:31 2013
@@ -17,11 +17,6 @@
  limitations under the License.
 -->
 
-<!-- $Id: solrconfig.xml 382610 2006-03-03 01:43:03Z yonik $
-     $Source$
-     $Name$
-  -->
-
 <config>
   <luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
   <jmx />



Mime
View raw message