lucenenet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From synhers...@apache.org
Subject [12/14] lucenenet git commit: Moving Lucene.Net.Facet tests to their appropriate place
Date Wed, 31 Dec 2014 19:12:18 GMT
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/68aa9728/src/Lucene.Net.Tests.Facet/SortedSet/TestSortedSetDocValuesFacets.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Facet/SortedSet/TestSortedSetDocValuesFacets.cs b/src/Lucene.Net.Tests.Facet/SortedSet/TestSortedSetDocValuesFacets.cs
new file mode 100644
index 0000000..091a2c8
--- /dev/null
+++ b/src/Lucene.Net.Tests.Facet/SortedSet/TestSortedSetDocValuesFacets.cs
@@ -0,0 +1,394 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+using Lucene.Net.Randomized;
+using Lucene.Net.Randomized.Generators;
+using NUnit.Framework;
+
+namespace Lucene.Net.Facet.SortedSet
+{
+
+    /*
+     * Licensed to the Apache Software Foundation (ASF) under one or more
+     * contributor license agreements.  See the NOTICE file distributed with
+     * this work for additional information regarding copyright ownership.
+     * The ASF licenses this file to You under the Apache License, Version 2.0
+     * (the "License"); you may not use this file except in compliance with
+     * the License.  You may obtain a copy of the License at
+     *
+     *     http://www.apache.org/licenses/LICENSE-2.0
+     *
+     * Unless required by applicable law or agreed to in writing, software
+     * distributed under the License is distributed on an "AS IS" BASIS,
+     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+     * See the License for the specific language governing permissions and
+     * limitations under the License.
+     */
+
+
+    using Document = Lucene.Net.Documents.Document;
+    using Field = Lucene.Net.Documents.Field;
+    using IndexReader = Lucene.Net.Index.IndexReader;
+    using RandomIndexWriter = Lucene.Net.Index.RandomIndexWriter;
+    using SlowCompositeReaderWrapper = Lucene.Net.Index.SlowCompositeReaderWrapper;
+    using Term = Lucene.Net.Index.Term;
+    using IndexSearcher = Lucene.Net.Search.IndexSearcher;
+    using MatchAllDocsQuery = Lucene.Net.Search.MatchAllDocsQuery;
+    using TermQuery = Lucene.Net.Search.TermQuery;
+    using TopDocs = Lucene.Net.Search.TopDocs;
+    using Directory = Lucene.Net.Store.Directory;
+    using IOUtils = Lucene.Net.Util.IOUtils;
+    using TestUtil = Lucene.Net.Util.TestUtil;
+    
+    [TestFixture]
+    public class TestSortedSetDocValuesFacets : FacetTestCase
+    {
+
+        // NOTE: TestDrillSideways.testRandom also sometimes
+        // randomly uses SortedSetDV
+        [Test]
+        public virtual void TestBasic()
+        {
+
+            AssumeTrue("Test requires SortedSetDV support", DefaultCodecSupportsSortedSet());
+            Directory dir = NewDirectory();
+
+            FacetsConfig config = new FacetsConfig();
+            config.SetMultiValued("a", true);
+            RandomIndexWriter writer = new RandomIndexWriter(Random(), dir);
+
+            Document doc = new Document();
+            doc.Add(new SortedSetDocValuesFacetField("a", "foo"));
+            doc.Add(new SortedSetDocValuesFacetField("a", "bar"));
+            doc.Add(new SortedSetDocValuesFacetField("a", "zoo"));
+            doc.Add(new SortedSetDocValuesFacetField("b", "baz"));
+            writer.AddDocument(config.Build(doc));
+            if (Random().NextBoolean())
+            {
+                writer.Commit();
+            }
+
+            doc = new Document();
+            doc.Add(new SortedSetDocValuesFacetField("a", "foo"));
+            writer.AddDocument(config.Build(doc));
+
+            // NRT open
+            IndexSearcher searcher = NewSearcher(writer.Reader);
+
+            // Per-top-reader state:
+            SortedSetDocValuesReaderState state = new DefaultSortedSetDocValuesReaderState(searcher.IndexReader);
+
+            FacetsCollector c = new FacetsCollector();
+
+            searcher.Search(new MatchAllDocsQuery(), c);
+
+            SortedSetDocValuesFacetCounts facets = new SortedSetDocValuesFacetCounts(state, c);
+
+            Assert.AreEqual("dim=a path=[] value=4 childCount=3\n  foo (2)\n  bar (1)\n  zoo (1)\n", facets.GetTopChildren(10, "a").ToString());
+            Assert.AreEqual("dim=b path=[] value=1 childCount=1\n  baz (1)\n", facets.GetTopChildren(10, "b").ToString());
+
+            // DrillDown:
+            DrillDownQuery q = new DrillDownQuery(config);
+            q.Add("a", "foo");
+            q.Add("b", "baz");
+            TopDocs hits = searcher.Search(q, 1);
+            Assert.AreEqual(1, hits.TotalHits);
+
+            IOUtils.Close(writer, searcher.IndexReader, dir);
+        }
+
+        // LUCENE-5090
+        [Test]
+        public virtual void TestStaleState()
+        {
+            AssumeTrue("Test requires SortedSetDV support", DefaultCodecSupportsSortedSet());
+            Directory dir = NewDirectory();
+
+            RandomIndexWriter writer = new RandomIndexWriter(Random(), dir);
+
+            FacetsConfig config = new FacetsConfig();
+
+            Document doc = new Document();
+            doc.Add(new SortedSetDocValuesFacetField("a", "foo"));
+            writer.AddDocument(config.Build(doc));
+
+            IndexReader r = writer.Reader;
+            SortedSetDocValuesReaderState state = new DefaultSortedSetDocValuesReaderState(r);
+
+            doc = new Document();
+            doc.Add(new SortedSetDocValuesFacetField("a", "bar"));
+            writer.AddDocument(config.Build(doc));
+
+            doc = new Document();
+            doc.Add(new SortedSetDocValuesFacetField("a", "baz"));
+            writer.AddDocument(config.Build(doc));
+
+            IndexSearcher searcher = NewSearcher(writer.Reader);
+
+            FacetsCollector c = new FacetsCollector();
+
+            searcher.Search(new MatchAllDocsQuery(), c);
+
+            try
+            {
+                new SortedSetDocValuesFacetCounts(state, c);
+                Fail("did not hit expected exception");
+            }
+            catch (IllegalStateException)
+            {
+                // expected
+            }
+
+            r.Dispose();
+            writer.Dispose();
+            searcher.IndexReader.Dispose();
+            dir.Dispose();
+        }
+
+        // LUCENE-5333
+        [Test]
+        public virtual void TestSparseFacets()
+        {
+            AssumeTrue("Test requires SortedSetDV support", DefaultCodecSupportsSortedSet());
+            Directory dir = NewDirectory();
+
+            RandomIndexWriter writer = new RandomIndexWriter(Random(), dir);
+
+            FacetsConfig config = new FacetsConfig();
+
+            Document doc = new Document();
+            doc.Add(new SortedSetDocValuesFacetField("a", "foo1"));
+            writer.AddDocument(config.Build(doc));
+
+            if (Random().NextBoolean())
+            {
+                writer.Commit();
+            }
+
+            doc = new Document();
+            doc.Add(new SortedSetDocValuesFacetField("a", "foo2"));
+            doc.Add(new SortedSetDocValuesFacetField("b", "bar1"));
+            writer.AddDocument(config.Build(doc));
+
+            if (Random().NextBoolean())
+            {
+                writer.Commit();
+            }
+
+            doc = new Document();
+            doc.Add(new SortedSetDocValuesFacetField("a", "foo3"));
+            doc.Add(new SortedSetDocValuesFacetField("b", "bar2"));
+            doc.Add(new SortedSetDocValuesFacetField("c", "baz1"));
+            writer.AddDocument(config.Build(doc));
+
+            // NRT open
+            IndexSearcher searcher = NewSearcher(writer.Reader);
+            writer.Dispose();
+
+            // Per-top-reader state:
+            SortedSetDocValuesReaderState state = new DefaultSortedSetDocValuesReaderState(searcher.IndexReader);
+
+            FacetsCollector c = new FacetsCollector();
+            searcher.Search(new MatchAllDocsQuery(), c);
+            SortedSetDocValuesFacetCounts facets = new SortedSetDocValuesFacetCounts(state, c);
+
+            // Ask for top 10 labels for any dims that have counts:
+            IList<FacetResult> results = facets.GetAllDims(10);
+
+            Assert.AreEqual(3, results.Count);
+            Assert.AreEqual("dim=a path=[] value=3 childCount=3\n  foo1 (1)\n  foo2 (1)\n  foo3 (1)\n", results[0].ToString());
+            Assert.AreEqual("dim=b path=[] value=2 childCount=2\n  bar1 (1)\n  bar2 (1)\n", results[1].ToString());
+            Assert.AreEqual("dim=c path=[] value=1 childCount=1\n  baz1 (1)\n", results[2].ToString());
+
+            searcher.IndexReader.Dispose();
+            dir.Dispose();
+        }
+
+        [Test]
+        public virtual void TestSomeSegmentsMissing()
+        {
+            AssumeTrue("Test requires SortedSetDV support", DefaultCodecSupportsSortedSet());
+            Directory dir = NewDirectory();
+
+            RandomIndexWriter writer = new RandomIndexWriter(Random(), dir);
+
+            FacetsConfig config = new FacetsConfig();
+
+            Document doc = new Document();
+            doc.Add(new SortedSetDocValuesFacetField("a", "foo1"));
+            writer.AddDocument(config.Build(doc));
+            writer.Commit();
+
+            doc = new Document();
+            writer.AddDocument(config.Build(doc));
+            writer.Commit();
+
+            doc = new Document();
+            doc.Add(new SortedSetDocValuesFacetField("a", "foo2"));
+            writer.AddDocument(config.Build(doc));
+            writer.Commit();
+
+            // NRT open
+            IndexSearcher searcher = NewSearcher(writer.Reader);
+            writer.Dispose();
+
+            // Per-top-reader state:
+            SortedSetDocValuesReaderState state = new DefaultSortedSetDocValuesReaderState(searcher.IndexReader);
+
+            FacetsCollector c = new FacetsCollector();
+            searcher.Search(new MatchAllDocsQuery(), c);
+            SortedSetDocValuesFacetCounts facets = new SortedSetDocValuesFacetCounts(state, c);
+
+            // Ask for top 10 labels for any dims that have counts:
+            Assert.AreEqual("dim=a path=[] value=2 childCount=2\n  foo1 (1)\n  foo2 (1)\n", facets.GetTopChildren(10, "a").ToString());
+
+            searcher.IndexReader.Dispose();
+            dir.Dispose();
+        }
+
+        [Test]
+        public virtual void TestSlowCompositeReaderWrapper()
+        {
+            AssumeTrue("Test requires SortedSetDV support", DefaultCodecSupportsSortedSet());
+            Directory dir = NewDirectory();
+
+            RandomIndexWriter writer = new RandomIndexWriter(Random(), dir);
+
+            FacetsConfig config = new FacetsConfig();
+
+            Document doc = new Document();
+            doc.Add(new SortedSetDocValuesFacetField("a", "foo1"));
+            writer.AddDocument(config.Build(doc));
+
+            writer.Commit();
+
+            doc = new Document();
+            doc.Add(new SortedSetDocValuesFacetField("a", "foo2"));
+            writer.AddDocument(config.Build(doc));
+
+            // NRT open
+            IndexSearcher searcher = new IndexSearcher(SlowCompositeReaderWrapper.Wrap(writer.Reader));
+
+            // Per-top-reader state:
+            SortedSetDocValuesReaderState state = new DefaultSortedSetDocValuesReaderState(searcher.IndexReader);
+
+            FacetsCollector c = new FacetsCollector();
+            searcher.Search(new MatchAllDocsQuery(), c);
+            Facets facets = new SortedSetDocValuesFacetCounts(state, c);
+
+            // Ask for top 10 labels for any dims that have counts:
+            Assert.AreEqual("dim=a path=[] value=2 childCount=2\n  foo1 (1)\n  foo2 (1)\n", facets.GetTopChildren(10, "a").ToString());
+
+            IOUtils.Close(writer, searcher.IndexReader, dir);
+        }
+
+
+        [Test]
+        public virtual void TestRandom()
+        {
+            AssumeTrue("Test requires SortedSetDV support", DefaultCodecSupportsSortedSet());
+            string[] tokens = GetRandomTokens(10);
+            Directory indexDir = NewDirectory();
+            Directory taxoDir = NewDirectory();
+
+            RandomIndexWriter w = new RandomIndexWriter(Random(), indexDir);
+            FacetsConfig config = new FacetsConfig();
+            int numDocs = AtLeast(1000);
+            int numDims = TestUtil.NextInt(Random(), 1, 7);
+            IList<TestDoc> testDocs = GetRandomDocs(tokens, numDocs, numDims);
+            foreach (TestDoc testDoc in testDocs)
+            {
+                Document doc = new Document();
+                doc.Add(NewStringField("content", testDoc.content, Field.Store.NO));
+                for (int j = 0; j < numDims; j++)
+                {
+                    if (testDoc.dims[j] != null)
+                    {
+                        doc.Add(new SortedSetDocValuesFacetField("dim" + j, testDoc.dims[j]));
+                    }
+                }
+                w.AddDocument(config.Build(doc));
+            }
+
+            // NRT open
+            IndexSearcher searcher = NewSearcher(w.Reader);
+
+            // Per-top-reader state:
+            SortedSetDocValuesReaderState state = new DefaultSortedSetDocValuesReaderState(searcher.IndexReader);
+
+            int iters = AtLeast(100);
+            for (int iter = 0; iter < iters; iter++)
+            {
+                string searchToken = tokens[Random().Next(tokens.Length)];
+                if (VERBOSE)
+                {
+                    Console.WriteLine("\nTEST: iter content=" + searchToken);
+                }
+                FacetsCollector fc = new FacetsCollector();
+                FacetsCollector.Search(searcher, new TermQuery(new Term("content", searchToken)), 10, fc);
+                Facets facets = new SortedSetDocValuesFacetCounts(state, fc);
+
+                // Slow, yet hopefully bug-free, faceting:
+                var expectedCounts = new List<Dictionary<string, int?>>();
+                for (int i = 0; i < numDims; i++)
+                {
+                    expectedCounts.Add(new Dictionary<string, int?>());
+                }
+
+                foreach (TestDoc doc in testDocs)
+                {
+                    if (doc.content.Equals(searchToken))
+                    {
+                        for (int j = 0; j < numDims; j++)
+                        {
+                            if (doc.dims[j] != null)
+                            {
+                                int? v; 
+                                
+                                if (!expectedCounts[j].TryGetValue(doc.dims[j],out v))
+                                {
+                                    expectedCounts[j][doc.dims[j]] = 1;
+                                }
+                                else
+                                {
+                                    expectedCounts[j][doc.dims[j]] = (int)v + 1;
+                                }
+                            }
+                        }
+                    }
+                }
+
+                IList<FacetResult> expected = new List<FacetResult>();
+                for (int i = 0; i < numDims; i++)
+                {
+                    IList<LabelAndValue> labelValues = new List<LabelAndValue>();
+                    int totCount = 0;
+                    foreach (KeyValuePair<string, int?> ent in expectedCounts[i])
+                    {
+                        labelValues.Add(new LabelAndValue(ent.Key, ent.Value.Value));
+                        totCount += ent.Value.Value;
+                    }
+                    SortLabelValues(labelValues);
+                    if (totCount > 0)
+                    {
+                        expected.Add(new FacetResult("dim" + i, new string[0], totCount, labelValues.ToArray(), labelValues.Count));
+                    }
+                }
+
+                // Sort by highest value, tie break by value:
+                SortFacetResults(expected);
+
+                IList<FacetResult> actual = facets.GetAllDims(10);
+
+                // Messy: fixup ties
+                //sortTies(actual);
+
+                Assert.AreEqual(expected, actual);
+            }
+
+            IOUtils.Close(w, searcher.IndexReader, indexDir, taxoDir);
+        }
+    }
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/68aa9728/src/Lucene.Net.Tests.Facet/Taxonomy/Directory/TestAddTaxonomy.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Facet/Taxonomy/Directory/TestAddTaxonomy.cs b/src/Lucene.Net.Tests.Facet/Taxonomy/Directory/TestAddTaxonomy.cs
new file mode 100644
index 0000000..df4f491
--- /dev/null
+++ b/src/Lucene.Net.Tests.Facet/Taxonomy/Directory/TestAddTaxonomy.cs
@@ -0,0 +1,323 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Threading;
+using System.Threading.Tasks;
+using Lucene.Net.Randomized.Generators;
+using Lucene.Net.Support;
+using NUnit.Framework;
+
+namespace Lucene.Net.Facet.Taxonomy.Directory
+{
+
+
+    using DiskOrdinalMap = Lucene.Net.Facet.Taxonomy.Directory.DirectoryTaxonomyWriter.DiskOrdinalMap;
+    using MemoryOrdinalMap = Lucene.Net.Facet.Taxonomy.Directory.DirectoryTaxonomyWriter.MemoryOrdinalMap;
+    using OrdinalMap = Lucene.Net.Facet.Taxonomy.Directory.DirectoryTaxonomyWriter.OrdinalMap;
+    using Directory = Lucene.Net.Store.Directory;
+    using IOUtils = Lucene.Net.Util.IOUtils;
+    using TestUtil = Lucene.Net.Util.TestUtil;
+
+    /*
+     * Licensed to the Apache Software Foundation (ASF) under one or more
+     * contributor license agreements.  See the NOTICE file distributed with
+     * this work for additional information regarding copyright ownership.
+     * The ASF licenses this file to You under the Apache License, Version 2.0
+     * (the "License"); you may not use this file except in compliance with
+     * the License.  You may obtain a copy of the License at
+     *
+     *     http://www.apache.org/licenses/LICENSE-2.0
+     *
+     * Unless required by applicable law or agreed to in writing, software
+     * distributed under the License is distributed on an "AS IS" BASIS,
+     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+     * See the License for the specific language governing permissions and
+     * limitations under the License.
+     */
+    [TestFixture]
+    public class TestAddTaxonomy : FacetTestCase
+    {
+        private void Dotest(int ncats, int range)
+        {
+            AtomicInteger numCats = new AtomicInteger(ncats);
+            Directory[] dirs = new Directory[2];
+            for (int i = 0; i < dirs.Length; i++)
+            {
+                dirs[i] = NewDirectory();
+                var tw = new DirectoryTaxonomyWriter(dirs[i]);
+                ThreadClass[] addThreads = new ThreadClass[4];
+                for (int j = 0; j < addThreads.Length; j++)
+                {
+                    addThreads[j] = new ThreadAnonymousInnerClassHelper(this, range, numCats, tw);
+                }
+
+                foreach (ThreadClass t in addThreads)
+                {
+                    t.Start();
+                }
+                foreach (ThreadClass t in addThreads)
+                {
+                    t.Join();
+                }
+
+                tw.Dispose();
+            }
+
+            var tw1 = new DirectoryTaxonomyWriter(dirs[0]);
+            OrdinalMap map = randomOrdinalMap();
+            tw1.AddTaxonomy(dirs[1], map);
+            tw1.Dispose();
+
+            validate(dirs[0], dirs[1], map);
+
+            IOUtils.Close(dirs);
+        }
+
+        private class ThreadAnonymousInnerClassHelper : ThreadClass
+        {
+            private readonly TestAddTaxonomy outerInstance;
+
+            private int range;
+            private AtomicInteger numCats;
+            private DirectoryTaxonomyWriter tw;
+
+            public ThreadAnonymousInnerClassHelper(TestAddTaxonomy outerInstance, int range, AtomicInteger numCats, DirectoryTaxonomyWriter tw)
+            {
+                this.outerInstance = outerInstance;
+                this.range = range;
+                this.numCats = numCats;
+                this.tw = tw;
+            }
+
+            public override void Run()
+            {
+                Random random = Random();
+                while (numCats.DecrementAndGet() > 0)
+                {
+                    string cat = Convert.ToString(random.Next(range));
+                    try
+                    {
+                        tw.AddCategory(new FacetLabel("a", cat));
+                    }
+                    catch (IOException e)
+                    {
+                        throw new Exception(e.Message, e);
+                    }
+                }
+            }
+        }
+
+
+        private OrdinalMap randomOrdinalMap()
+        {
+            if (Random().NextBoolean())
+            {
+                return new DiskOrdinalMap("taxoMap");
+            }
+            else
+            {
+                return new MemoryOrdinalMap();
+            }
+        }
+
+        private void validate(Directory dest, Directory src, OrdinalMap ordMap)
+        {
+            var destTr = new DirectoryTaxonomyReader(dest);
+            try
+            {
+                int destSize = destTr.Size;
+                var srcTR = new DirectoryTaxonomyReader(src);
+                try
+                {
+                    var map = ordMap.Map;
+
+                    // validate taxo sizes
+                    int srcSize = srcTR.Size;
+                    Assert.True(destSize >= srcSize, "destination taxonomy expected to be larger than source; dest=" + destSize + " src=" + srcSize);
+
+                    // validate that all source categories exist in destination, and their
+                    // ordinals are as expected.
+                    for (int j = 1; j < srcSize; j++)
+                    {
+                        FacetLabel cp = srcTR.GetPath(j);
+                        int destOrdinal = destTr.GetOrdinal(cp);
+                        Assert.True(destOrdinal > 0, cp + " not found in destination");
+                        Assert.AreEqual(destOrdinal, map[j]);
+                    }
+                }
+                finally
+                {
+                    ((TaxonomyReader)srcTR).Dispose(true);
+                }
+            }
+            finally
+            {
+                ((TaxonomyReader)destTr).Dispose(true);
+            }
+        }
+
+        [Test]
+        public virtual void TestAddEmpty()
+        {
+            Directory dest = NewDirectory();
+            var destTW = new DirectoryTaxonomyWriter(dest);
+            destTW.AddCategory(new FacetLabel("Author", "Rob Pike"));
+            destTW.AddCategory(new FacetLabel("Aardvarks", "Bob"));
+            destTW.Commit();
+
+            Directory src = NewDirectory();
+            (new DirectoryTaxonomyWriter(src)).Dispose(); // create an empty taxonomy
+
+            OrdinalMap map = randomOrdinalMap();
+            destTW.AddTaxonomy(src, map);
+            destTW.Dispose();
+
+            validate(dest, src, map);
+
+            IOUtils.Close(dest, src);
+        }
+
+        [Test]
+        public virtual void TestAddToEmpty()
+        {
+            Directory dest = NewDirectory();
+
+            Directory src = NewDirectory();
+            DirectoryTaxonomyWriter srcTW = new DirectoryTaxonomyWriter(src);
+            srcTW.AddCategory(new FacetLabel("Author", "Rob Pike"));
+            srcTW.AddCategory(new FacetLabel("Aardvarks", "Bob"));
+            srcTW.Dispose();
+
+            DirectoryTaxonomyWriter destTW = new DirectoryTaxonomyWriter(dest);
+            OrdinalMap map = randomOrdinalMap();
+            destTW.AddTaxonomy(src, map);
+            destTW.Dispose();
+
+            validate(dest, src, map);
+
+            IOUtils.Close(dest, src);
+        }
+
+        // A more comprehensive and big random test.
+        [Test]
+        public virtual void TestBig()
+        {
+            Dotest(200, 10000);
+            Dotest(1000, 20000);
+            Dotest(400000, 1000000);
+        }
+
+        // a reasonable random test
+        [Test]
+        public virtual void TestMedium()
+        {
+            Random random = Random();
+            int numTests = AtLeast(3);
+            for (int i = 0; i < numTests; i++)
+            {
+                Dotest(TestUtil.NextInt(random, 2, 100), TestUtil.NextInt(random, 100, 1000));
+            }
+        }
+
+        [Test]
+        public virtual void TestSimple()
+        {
+            Directory dest = NewDirectory();
+            var tw1 = new DirectoryTaxonomyWriter(dest);
+            tw1.AddCategory(new FacetLabel("Author", "Mark Twain"));
+            tw1.AddCategory(new FacetLabel("Animals", "Dog"));
+            tw1.AddCategory(new FacetLabel("Author", "Rob Pike"));
+
+            Directory src = NewDirectory();
+            var tw2 = new DirectoryTaxonomyWriter(src);
+            tw2.AddCategory(new FacetLabel("Author", "Rob Pike"));
+            tw2.AddCategory(new FacetLabel("Aardvarks", "Bob"));
+            tw2.Dispose();
+
+            OrdinalMap map = randomOrdinalMap();
+
+            tw1.AddTaxonomy(src, map);
+            tw1.Dispose();
+
+            validate(dest, src, map);
+
+            IOUtils.Close(dest, src);
+        }
+
+        [Test]
+        public virtual void TestConcurrency()
+        {
+            // tests that addTaxonomy and addCategory work in parallel
+            int numCategories = AtLeast(10000);
+
+            // build an input taxonomy index
+            Directory src = NewDirectory();
+            var tw = new DirectoryTaxonomyWriter(src);
+            for (int i = 0; i < numCategories; i++)
+            {
+                tw.AddCategory(new FacetLabel("a", Convert.ToString(i)));
+            }
+            tw.Dispose();
+
+            // now add the taxonomy to an empty taxonomy, while adding the categories
+            // again, in parallel -- in the end, no duplicate categories should exist.
+            Directory dest = NewDirectory();
+            var destTw = new DirectoryTaxonomyWriter(dest);
+            ThreadClass t = new ThreadAnonymousInnerClassHelper2(this, numCategories, destTw);
+            t.Start();
+
+            OrdinalMap map = new MemoryOrdinalMap();
+            destTw.AddTaxonomy(src, map);
+            t.Join();
+            destTw.Dispose();
+
+            // now validate
+
+            var dtr = new DirectoryTaxonomyReader(dest);
+            // +2 to account for the root category + "a"
+            Assert.AreEqual(numCategories + 2, dtr.Size);
+            var categories = new HashSet<FacetLabel>();
+            for (int i = 1; i < dtr.Size; i++)
+            {
+                FacetLabel cat = dtr.GetPath(i);
+                Assert.True(categories.Add(cat), "category " + cat + " already existed");
+            }
+            dtr.Dispose();
+
+            IOUtils.Close(src, dest);
+        }
+
+        private class ThreadAnonymousInnerClassHelper2 : ThreadClass
+        {
+            private readonly TestAddTaxonomy outerInstance;
+
+            private int numCategories;
+            private Lucene.Net.Facet.Taxonomy.Directory.DirectoryTaxonomyWriter destTW;
+
+            public ThreadAnonymousInnerClassHelper2(TestAddTaxonomy outerInstance, int numCategories, Lucene.Net.Facet.Taxonomy.Directory.DirectoryTaxonomyWriter destTW)
+            {
+                this.outerInstance = outerInstance;
+                this.numCategories = numCategories;
+                this.destTW = destTW;
+            }
+
+            public override void Run()
+            {
+                for (int i = 0; i < numCategories; i++)
+                {
+                    try
+                    {
+                        destTW.AddCategory(new FacetLabel("a", Convert.ToString(i)));
+                    }
+                    catch (IOException e)
+                    {
+                        // shouldn't happen - if it does, let the test fail on uncaught exception.
+                        throw new Exception(e.Message, e);
+                    }
+                }
+            }
+        }
+
+    }
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/68aa9728/src/Lucene.Net.Tests.Facet/Taxonomy/Directory/TestConcurrentFacetedIndexing.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Facet/Taxonomy/Directory/TestConcurrentFacetedIndexing.cs b/src/Lucene.Net.Tests.Facet/Taxonomy/Directory/TestConcurrentFacetedIndexing.cs
new file mode 100644
index 0000000..5ee17a6
--- /dev/null
+++ b/src/Lucene.Net.Tests.Facet/Taxonomy/Directory/TestConcurrentFacetedIndexing.cs
@@ -0,0 +1,228 @@
+using System;
+using System.Collections.Concurrent;
+using System.IO;
+using System.Threading;
+using Lucene.Net.Support;
+using NUnit.Framework;
+
+namespace Lucene.Net.Facet.Taxonomy.Directory
+{
+
+
+    using Document = Lucene.Net.Documents.Document;
+    using TaxonomyWriterCache = Lucene.Net.Facet.Taxonomy.WriterCache.TaxonomyWriterCache;
+    using Cl2oTaxonomyWriterCache = Lucene.Net.Facet.Taxonomy.WriterCache.Cl2oTaxonomyWriterCache;
+    using LruTaxonomyWriterCache = Lucene.Net.Facet.Taxonomy.WriterCache.LruTaxonomyWriterCache;
+    using IndexWriter = Lucene.Net.Index.IndexWriter;
+    using OpenMode = Lucene.Net.Index.IndexWriterConfig.OpenMode_e;
+    using Directory = Lucene.Net.Store.Directory;
+    using IOUtils = Lucene.Net.Util.IOUtils;
+
+    /*
+     * Licensed to the Apache Software Foundation (ASF) under one or more
+     * contributor license agreements.  See the NOTICE file distributed with
+     * this work for additional information regarding copyright ownership.
+     * The ASF licenses this file to You under the Apache License, Version 2.0
+     * (the "License"); you may not use this file except in compliance with
+     * the License.  You may obtain a copy of the License at
+     *
+     *     http://www.apache.org/licenses/LICENSE-2.0
+     *
+     * Unless required by applicable law or agreed to in writing, software
+     * distributed under the License is distributed on an "AS IS" BASIS,
+     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+     * See the License for the specific language governing permissions and
+     * limitations under the License.
+     */
+    [TestFixture]
+    public class TestConcurrentFacetedIndexing : FacetTestCase
+    {
+
+        // A No-Op TaxonomyWriterCache which always discards all given categories, and
+        // always returns true in put(), to indicate some cache entries were cleared.
+        private static TaxonomyWriterCache NO_OP_CACHE = new TaxonomyWriterCacheAnonymousInnerClassHelper();
+
+        private class TaxonomyWriterCacheAnonymousInnerClassHelper : TaxonomyWriterCache
+        {
+            public TaxonomyWriterCacheAnonymousInnerClassHelper()
+            {
+            }
+
+
+            public virtual void Close()
+            {
+            }
+            public virtual int Get(FacetLabel categoryPath)
+            {
+                return -1;
+            }
+            public virtual bool Put(FacetLabel categoryPath, int ordinal)
+            {
+                return true;
+            }
+            public virtual bool Full
+            {
+                get
+                {
+                    return true;
+                }
+            }
+            public virtual void Clear()
+            {
+            }
+
+        }
+
+        internal static FacetField NewCategory()
+        {
+            Random r = Random();
+            string l1 = "l1." + r.Next(10); // l1.0-l1.9 (10 categories)
+            string l2 = "l2." + r.Next(30); // l2.0-l2.29 (30 categories)
+            string l3 = "l3." + r.Next(100); // l3.0-l3.99 (100 categories)
+            return new FacetField(l1, l2, l3);
+        }
+
+        internal static TaxonomyWriterCache NewTaxoWriterCache(int ndocs)
+        {
+            double d = Random().NextDouble();
+            if (d < 0.7)
+            {
+                // this is the fastest, yet most memory consuming
+                return new Cl2oTaxonomyWriterCache(1024, 0.15f, 3);
+            }
+            else if (TEST_NIGHTLY && d > 0.98)
+            {
+                // this is the slowest, but tests the writer concurrency when no caching is done.
+                // only pick it during NIGHTLY tests, and even then, with very low chances.
+                return NO_OP_CACHE;
+            }
+            else
+            {
+                // this is slower than CL2O, but less memory consuming, and exercises finding categories on disk too.
+                return new LruTaxonomyWriterCache(ndocs / 10);
+            }
+        }
+
+        [Test]
+        public virtual void TestConcurrency()
+        {
+            AtomicInteger numDocs = new AtomicInteger(AtLeast(10000));
+            Directory indexDir = NewDirectory();
+            Directory taxoDir = NewDirectory();
+            ConcurrentDictionary<string, string> values = new ConcurrentDictionary<string, string>();
+            IndexWriter iw = new IndexWriter(indexDir, NewIndexWriterConfig(TEST_VERSION_CURRENT, null));
+            var tw = new DirectoryTaxonomyWriter(taxoDir, OpenMode.CREATE, NewTaxoWriterCache(numDocs.Get()));
+            ThreadClass[] indexThreads = new ThreadClass[AtLeast(4)];
+            FacetsConfig config = new FacetsConfig();
+            for (int i = 0; i < 10; i++)
+            {
+                config.SetHierarchical("l1." + i, true);
+                config.SetMultiValued("l1." + i, true);
+            }
+
+            for (int i = 0; i < indexThreads.Length; i++)
+            {
+                indexThreads[i] = new ThreadAnonymousInnerClassHelper(this, numDocs, values, iw, tw, config);
+            }
+
+            foreach (ThreadClass t in indexThreads)
+            {
+                t.Start();
+            }
+            foreach (ThreadClass t in indexThreads)
+            {
+                t.Join();
+            }
+
+            var tr = new DirectoryTaxonomyReader(tw);
+            // +1 for root category
+            if (values.Count + 1 != tr.Size)
+            {
+                foreach (string value in values.Keys)
+                {
+                    FacetLabel label = new FacetLabel(FacetsConfig.StringToPath(value));
+                    if (tr.GetOrdinal(label) == -1)
+                    {
+                        Console.WriteLine("FAIL: path=" + label + " not recognized");
+                    }
+                }
+                Fail("mismatch number of categories");
+            }
+            int[] parents = tr.ParallelTaxonomyArrays.Parents();
+            foreach (string cat in values.Keys)
+            {
+                FacetLabel cp = new FacetLabel(FacetsConfig.StringToPath(cat));
+                Assert.True(tr.GetOrdinal(cp) > 0, "category not found " + cp);
+                int level = cp.Length;
+                int parentOrd = 0; // for root, parent is always virtual ROOT (ord=0)
+                FacetLabel path = null;
+                for (int i = 0; i < level; i++)
+                {
+                    path = cp.Subpath(i + 1);
+                    int ord = tr.GetOrdinal(path);
+                    Assert.AreEqual(parentOrd, parents[ord], "invalid parent for cp=" + path);
+                    parentOrd = ord; // next level should have this parent
+                }
+            }
+
+            IOUtils.Close(tw, iw, tr, taxoDir, indexDir);
+        }
+
+        private class ThreadAnonymousInnerClassHelper : ThreadClass
+        {
+            private readonly TestConcurrentFacetedIndexing outerInstance;
+
+            private AtomicInteger numDocs;
+            private ConcurrentDictionary<string, string> values;
+            private IndexWriter iw;
+            private Lucene.Net.Facet.Taxonomy.Directory.DirectoryTaxonomyWriter tw;
+            private FacetsConfig config;
+
+            public ThreadAnonymousInnerClassHelper(TestConcurrentFacetedIndexing outerInstance, AtomicInteger numDocs, ConcurrentDictionary<string, string> values, IndexWriter iw, Lucene.Net.Facet.Taxonomy.Directory.DirectoryTaxonomyWriter tw, FacetsConfig config)
+            {
+                this.outerInstance = outerInstance;
+                this.numDocs = numDocs;
+                this.values = values;
+                this.iw = iw;
+                this.tw = tw;
+                this.config = config;
+            }
+
+
+            public override void Run()
+            {
+                Random random = Random();
+                while (numDocs.DecrementAndGet() > 0)
+                {
+                    try
+                    {
+                        Document doc = new Document();
+                        int numCats = random.Next(3) + 1; // 1-3
+                        while (numCats-- > 0)
+                        {
+                            FacetField ff = NewCategory();
+                            doc.Add(ff);
+
+                            FacetLabel label = new FacetLabel(ff.dim, ff.path);
+                            // add all prefixes to values
+                            int level = label.Length;
+                            while (level > 0)
+                            {
+                                string s = FacetsConfig.PathToString(label.Components, level);
+                                values[s] = s;
+                                --level;
+                            }
+                        }
+                        iw.AddDocument(config.Build(tw, doc));
+                    }
+                    catch (IOException e)
+                    {
+                        throw new Exception(e.Message, e);
+                    }
+                }
+            }
+        }
+
+    }
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/68aa9728/src/Lucene.Net.Tests.Facet/Taxonomy/Directory/TestDirectoryTaxonomyReader.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Facet/Taxonomy/Directory/TestDirectoryTaxonomyReader.cs b/src/Lucene.Net.Tests.Facet/Taxonomy/Directory/TestDirectoryTaxonomyReader.cs
new file mode 100644
index 0000000..15f30df
--- /dev/null
+++ b/src/Lucene.Net.Tests.Facet/Taxonomy/Directory/TestDirectoryTaxonomyReader.cs
@@ -0,0 +1,624 @@
+using System;
+using System.Collections.Generic;
+using Lucene.Net.Randomized.Generators;
+using Lucene.Net.Support;
+using NUnit.Framework;
+
+namespace Lucene.Net.Facet.Taxonomy.Directory
+{
+
+
+    using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer;
+    using ChildrenIterator = Lucene.Net.Facet.Taxonomy.TaxonomyReader.ChildrenIterator;
+    using IndexWriter = Lucene.Net.Index.IndexWriter;
+    using IndexWriterConfig = Lucene.Net.Index.IndexWriterConfig;
+    using OpenMode = Lucene.Net.Index.IndexWriterConfig.OpenMode_e;
+    using LogByteSizeMergePolicy = Lucene.Net.Index.LogByteSizeMergePolicy;
+    using LogMergePolicy = Lucene.Net.Index.LogMergePolicy;
+    using AlreadyClosedException = Lucene.Net.Store.AlreadyClosedException;
+    using Directory = Lucene.Net.Store.Directory;
+    using RAMDirectory = Lucene.Net.Store.RAMDirectory;
+    using IOUtils = Lucene.Net.Util.IOUtils;
+
+    /*
+     * Licensed to the Apache Software Foundation (ASF) under one or more
+     * contributor license agreements.  See the NOTICE file distributed with
+     * this work for additional information regarding copyright ownership.
+     * The ASF licenses this file to You under the Apache License, Version 2.0
+     * (the "License"); you may not use this file except in compliance with
+     * the License.  You may obtain a copy of the License at
+     *
+     *     http://www.apache.org/licenses/LICENSE-2.0
+     *
+     * Unless required by applicable law or agreed to in writing, software
+     * distributed under the License is distributed on an "AS IS" BASIS,
+     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+     * See the License for the specific language governing permissions and
+     * limitations under the License.
+     */
+
+    public class TestDirectoryTaxonomyReader : FacetTestCase
+    {
+
+        [Test]
+        public virtual void TestCloseAfterIncRef()
+        {
+            Directory dir = NewDirectory();
+            var ltw = new DirectoryTaxonomyWriter(dir);
+            ltw.AddCategory(new FacetLabel("a"));
+            ltw.Dispose();
+
+            DirectoryTaxonomyReader ltr = new DirectoryTaxonomyReader(dir);
+            ltr.IncRef();
+            ltr.Dispose();
+
+            // should not fail as we IncRef() before close
+            var tmpSie = ltr.Size;
+            ltr.DecRef();
+
+            dir.Dispose();
+        }
+
+        [Test]
+        public virtual void TestCloseTwice()
+        {
+            Directory dir = NewDirectory();
+            var ltw = new DirectoryTaxonomyWriter(dir);
+            ltw.AddCategory(new FacetLabel("a"));
+            ltw.Dispose();
+
+            var ltr = new DirectoryTaxonomyReader(dir);
+            (ltr).Dispose();
+            (ltr).Dispose(); // no exception should be thrown
+
+            dir.Dispose();
+        }
+
+        [Test]
+        public virtual void TestOpenIfChangedResult()
+        {
+            Directory dir = null;
+            DirectoryTaxonomyWriter ltw = null;
+            DirectoryTaxonomyReader ltr = null;
+
+            try
+            {
+                dir = NewDirectory();
+                ltw = new DirectoryTaxonomyWriter(dir);
+
+                ltw.AddCategory(new FacetLabel("a"));
+                ltw.Commit();
+
+                ltr = new DirectoryTaxonomyReader(dir);
+                Assert.Null(TaxonomyReader.OpenIfChanged(ltr), "Nothing has changed");
+
+                ltw.AddCategory(new FacetLabel("b"));
+                ltw.Commit();
+
+                DirectoryTaxonomyReader newtr = TaxonomyReader.OpenIfChanged(ltr);
+                Assert.NotNull(newtr, "changes were committed");
+                Assert.Null(TaxonomyReader.OpenIfChanged(newtr), "Nothing has changed");
+                (newtr).Dispose();
+            }
+            finally
+            {
+                IOUtils.Close(ltw, ltr, dir);
+            }
+        }
+
+        [Test]
+        public virtual void TestAlreadyClosed()
+        {
+            Directory dir = NewDirectory();
+            var ltw = new DirectoryTaxonomyWriter(dir);
+            ltw.AddCategory(new FacetLabel("a"));
+            ltw.Dispose();
+
+            var ltr = new DirectoryTaxonomyReader(dir);
+            ltr.Dispose();
+            try
+            {
+                var tmpSize = ltr.Size;
+                Fail("An AlreadyClosedException should have been thrown here");
+            }
+            catch (AlreadyClosedException)
+            {
+                // good!
+            }
+            dir.Dispose();
+        }
+
+        /// <summary>
+        /// recreating a taxonomy should work well with a freshly opened taxonomy reader 
+        /// </summary>
+        [Test]
+        public virtual void TestFreshReadRecreatedTaxonomy()
+        {
+            doTestReadRecreatedTaxonomy(Random(), true);
+        }
+
+        [Test]
+        public virtual void TestOpenIfChangedReadRecreatedTaxonomy()
+        {
+            doTestReadRecreatedTaxonomy(Random(), false);
+        }
+        
+        private void doTestReadRecreatedTaxonomy(Random random, bool closeReader)
+        {
+            Directory dir = null;
+            TaxonomyWriter tw = null;
+            TaxonomyReader tr = null;
+
+            // prepare a few categories
+            int n = 10;
+            FacetLabel[] cp = new FacetLabel[n];
+            for (int i = 0; i < n; i++)
+            {
+                cp[i] = new FacetLabel("a", Convert.ToString(i));
+            }
+
+            try
+            {
+                dir = NewDirectory();
+
+                tw = new DirectoryTaxonomyWriter(dir);
+                tw.AddCategory(new FacetLabel("a"));
+                tw.Dispose();
+
+                tr = new DirectoryTaxonomyReader(dir);
+                int baseNumCategories = tr.Size;
+
+                for (int i = 0; i < n; i++)
+                {
+                    int k = random.Next(n);
+                    tw = new DirectoryTaxonomyWriter(dir, IndexWriterConfig.OpenMode_e.CREATE);
+                    for (int j = 0; j <= k; j++)
+                    {
+                        tw.AddCategory(cp[j]);
+                    }
+                    tw.Dispose();
+                    if (closeReader)
+                    {
+                        tr.Dispose(true);
+                        tr = new DirectoryTaxonomyReader(dir);
+                    }
+                    else
+                    {
+                        var newtr = TaxonomyReader.OpenIfChanged(tr);
+                        Assert.NotNull(newtr);
+                        tr.Dispose(true);
+                        tr = newtr;
+                    }
+                    Assert.AreEqual(baseNumCategories + 1 + k, tr.Size, "Wrong #categories in taxonomy (i=" + i + ", k=" + k + ")");
+                }
+            }
+            finally
+            {
+                IOUtils.Close(tr as DirectoryTaxonomyReader, tw, dir);
+            }
+        }
+
+        [Test]
+        public virtual void TestOpenIfChangedAndRefCount()
+        {
+            Directory dir = new RAMDirectory(); // no need for random directories here
+
+            var taxoWriter = new DirectoryTaxonomyWriter(dir);
+            taxoWriter.AddCategory(new FacetLabel("a"));
+            taxoWriter.Commit();
+
+            var taxoReader = new DirectoryTaxonomyReader(dir);
+            Assert.AreEqual(1, taxoReader.RefCount, "wrong refCount");
+
+            taxoReader.IncRef();
+            Assert.AreEqual(2, taxoReader.RefCount, "wrong refCount");
+
+            taxoWriter.AddCategory(new FacetLabel("a", "b"));
+            taxoWriter.Commit();
+            var newtr = TaxonomyReader.OpenIfChanged(taxoReader);
+            Assert.NotNull(newtr);
+            taxoReader.Dispose();
+            taxoReader = newtr;
+            Assert.AreEqual(1, taxoReader.RefCount, "wrong refCount");
+
+            taxoWriter.Dispose();
+            taxoReader.Dispose();
+            dir.Dispose();
+        }
+
+        [Test]
+        public virtual void TestOpenIfChangedManySegments()
+        {
+            // test openIfChanged() when the taxonomy contains many segments
+            Directory dir = NewDirectory();
+
+            DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriterAnonymousInnerClassHelper(this, dir);
+            var reader = new DirectoryTaxonomyReader(writer);
+
+            int numRounds = Random().Next(10) + 10;
+            int numCategories = 1; // one for root
+            for (int i = 0; i < numRounds; i++)
+            {
+                int numCats = Random().Next(4) + 1;
+                for (int j = 0; j < numCats; j++)
+                {
+                    writer.AddCategory(new FacetLabel(Convert.ToString(i), Convert.ToString(j)));
+                }
+                numCategories += numCats + 1; // one for round-parent
+                var newtr = TaxonomyReader.OpenIfChanged(reader);
+                Assert.NotNull(newtr);
+                reader.Dispose();
+                reader = newtr;
+
+                // assert categories
+                Assert.AreEqual(numCategories, reader.Size);
+                int roundOrdinal = reader.GetOrdinal(new FacetLabel(Convert.ToString(i)));
+                int[] parents = reader.ParallelTaxonomyArrays.Parents();
+                Assert.AreEqual(0, parents[roundOrdinal]); // round's parent is root
+                for (int j = 0; j < numCats; j++)
+                {
+                    int ord = reader.GetOrdinal(new FacetLabel(Convert.ToString(i), Convert.ToString(j)));
+                    Assert.AreEqual(roundOrdinal, parents[ord]); // round's parent is root
+                }
+            }
+
+            reader.Dispose();
+            writer.Dispose();
+            dir.Dispose();
+        }
+
+        private class DirectoryTaxonomyWriterAnonymousInnerClassHelper : DirectoryTaxonomyWriter
+        {
+            private readonly TestDirectoryTaxonomyReader outerInstance;
+
+            public DirectoryTaxonomyWriterAnonymousInnerClassHelper(TestDirectoryTaxonomyReader outerInstance, Directory dir)
+                : base(dir)
+            {
+                this.outerInstance = outerInstance;
+            }
+
+            protected override IndexWriterConfig CreateIndexWriterConfig(IndexWriterConfig.OpenMode_e openMode)
+            {
+                IndexWriterConfig conf = base.CreateIndexWriterConfig(openMode);
+                LogMergePolicy lmp = (LogMergePolicy)conf.MergePolicy;
+                lmp.MergeFactor = 2;
+                return conf;
+            }
+        }
+
+        [Test]
+        public virtual void TestOpenIfChangedMergedSegment()
+        {
+            // test openIfChanged() when all index segments were merged - used to be
+            // a bug in ParentArray, caught by testOpenIfChangedManySegments - only
+            // this test is not random
+            Directory dir = NewDirectory();
+
+            // hold onto IW to forceMerge
+            // note how we don't close it, since DTW will close it.
+            IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergePolicy(new LogByteSizeMergePolicy()));
+            var writer = new DirectoryTaxonomyWriterAnonymousInnerClassHelper2(this, dir, iw) as DirectoryTaxonomyWriter;
+
+            var reader = new DirectoryTaxonomyReader(writer);
+            Assert.AreEqual(1, reader.Size);
+            Assert.AreEqual(1, reader.ParallelTaxonomyArrays.Parents().Length);
+
+            // add category and call forceMerge -- this should flush IW and merge segments down to 1
+            // in ParentArray.initFromReader, this used to fail assuming there are no parents.
+            writer.AddCategory(new FacetLabel("1"));
+            iw.ForceMerge(1);
+
+            // now calling openIfChanged should trip on the bug
+            var newtr = TaxonomyReader.OpenIfChanged(reader);
+            Assert.NotNull(newtr);
+            reader.Dispose();
+            reader = newtr;
+            Assert.AreEqual(2, reader.Size);
+            Assert.AreEqual(2, reader.ParallelTaxonomyArrays.Parents().Length);
+
+            reader.Dispose();
+            writer.Dispose();
+            dir.Dispose();
+        }
+
+        private class DirectoryTaxonomyWriterAnonymousInnerClassHelper2 : DirectoryTaxonomyWriter
+        {
+            private readonly TestDirectoryTaxonomyReader outerInstance;
+
+            private IndexWriter iw;
+            private IndexWriterConfig config;
+
+            public DirectoryTaxonomyWriterAnonymousInnerClassHelper2(TestDirectoryTaxonomyReader outerInstance, Directory dir, IndexWriter iw) : base(dir)
+            {
+                this.outerInstance = outerInstance;
+                this.iw = iw;
+            }
+
+            protected override IndexWriter OpenIndexWriter(Directory directory, IndexWriterConfig config) 
+            {   
+                return iw;
+            }
+        }
+
+        [Test]
+        public virtual void TestOpenIfChangedNoChangesButSegmentMerges()
+        {
+            // test openIfChanged() when the taxonomy hasn't really changed, but segments
+            // were merged. The NRT reader will be reopened, and ParentArray used to assert
+            // that the new reader contains more ordinals than were given from the old
+            // TaxReader version
+            Directory dir = NewDirectory();
+
+            // hold onto IW to forceMerge
+            // note how we don't close it, since DTW will close it.
+            var iw = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergePolicy(new LogByteSizeMergePolicy()));
+            DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriterAnonymousInnerClassHelper3(this, dir, iw);
+
+
+            // add a category so that the following DTR open will cause a flush and 
+            // a new segment will be created
+            writer.AddCategory(new FacetLabel("a"));
+
+            var reader = new DirectoryTaxonomyReader(writer);
+            Assert.AreEqual(2, reader.Size);
+            Assert.AreEqual(2, reader.ParallelTaxonomyArrays.Parents().Length);
+
+            // merge all the segments so that NRT reader thinks there's a change 
+            iw.ForceMerge(1);
+
+            // now calling openIfChanged should trip on the wrong assert in ParetArray's ctor
+            var newtr = TaxonomyReader.OpenIfChanged(reader);
+            Assert.NotNull(newtr);
+            reader.Dispose();
+            reader = newtr;
+            Assert.AreEqual(2, reader.Size);
+            Assert.AreEqual(2, reader.ParallelTaxonomyArrays.Parents().Length);
+
+            reader.Dispose();
+            writer.Dispose();
+            dir.Dispose();
+        }
+
+        private class DirectoryTaxonomyWriterAnonymousInnerClassHelper3 : DirectoryTaxonomyWriter
+        {
+            private readonly TestDirectoryTaxonomyReader outerInstance;
+
+            private IndexWriter iw;
+
+            public DirectoryTaxonomyWriterAnonymousInnerClassHelper3(TestDirectoryTaxonomyReader outerInstance, Directory dir, IndexWriter iw)
+                : base(dir)
+            {
+                this.outerInstance = outerInstance;
+                this.iw = iw;
+            }
+
+            protected override IndexWriter OpenIndexWriter(Directory directory, IndexWriterConfig config)
+            {
+                return iw;
+            }
+        }
+
+        [Test]
+        public virtual void TestOpenIfChangedReuseAfterRecreate()
+        {
+            // tests that if the taxonomy is recreated, no data is reused from the previous taxonomy
+            Directory dir = NewDirectory();
+            DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir);
+            FacetLabel cp_a = new FacetLabel("a");
+            writer.AddCategory(cp_a);
+            writer.Dispose();
+
+            DirectoryTaxonomyReader r1 = new DirectoryTaxonomyReader(dir);
+            // fill r1's caches
+            Assert.AreEqual(1, r1.GetOrdinal(cp_a));
+            Assert.AreEqual(cp_a, r1.GetPath(1));
+
+            // now recreate, add a different category
+            writer = new DirectoryTaxonomyWriter(dir, IndexWriterConfig.OpenMode_e.CREATE);
+            FacetLabel cp_b = new FacetLabel("b");
+            writer.AddCategory(cp_b);
+            writer.Dispose();
+
+            DirectoryTaxonomyReader r2 = TaxonomyReader.OpenIfChanged(r1);
+            Assert.NotNull(r2);
+
+            // fill r2's caches
+            Assert.AreEqual(1, r2.GetOrdinal(cp_b));
+            Assert.AreEqual(cp_b, r2.GetPath(1));
+
+            // check that r1 doesn't see cp_b
+            Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, r1.GetOrdinal(cp_b));
+            Assert.AreEqual(cp_a, r1.GetPath(1));
+
+            // check that r2 doesn't see cp_a
+            Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, r2.GetOrdinal(cp_a));
+            Assert.AreEqual(cp_b, r2.GetPath(1));
+
+            (r2).Dispose();
+            (r1).Dispose();
+            dir.Dispose();
+        }
+
+        [Test]
+        public virtual void TestOpenIfChangedReuse()
+        {
+            // test the reuse of data from the old DTR instance
+            foreach (bool nrt in new bool[] { false, true })
+            {
+                Directory dir = NewDirectory();
+                DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir);
+
+                FacetLabel cp_a = new FacetLabel("a");
+                writer.AddCategory(cp_a);
+                if (!nrt)
+                {
+                    writer.Commit();
+                }
+
+                DirectoryTaxonomyReader r1 = nrt ? new DirectoryTaxonomyReader(writer) : new DirectoryTaxonomyReader(dir);
+                // fill r1's caches
+                Assert.AreEqual(1, r1.GetOrdinal(cp_a));
+                Assert.AreEqual(cp_a, r1.GetPath(1));
+
+                FacetLabel cp_b = new FacetLabel("b");
+                writer.AddCategory(cp_b);
+                if (!nrt)
+                {
+                    writer.Commit();
+                }
+
+                DirectoryTaxonomyReader r2 = TaxonomyReader.OpenIfChanged(r1);
+                Assert.NotNull(r2);
+
+                // add r2's categories to the caches
+                Assert.AreEqual(2, r2.GetOrdinal(cp_b));
+                Assert.AreEqual(cp_b, r2.GetPath(2));
+
+                // check that r1 doesn't see cp_b
+                Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, r1.GetOrdinal(cp_b));
+                Assert.Null(r1.GetPath(2));
+
+                (r1).Dispose();
+                (r2).Dispose();
+                writer.Dispose();
+                dir.Dispose();
+            }
+        }
+
+        [Test]
+        public virtual void TestOpenIfChangedReplaceTaxonomy()
+        {
+            // test openIfChanged when replaceTaxonomy is called, which is equivalent to recreate
+            // only can work with NRT as well
+            Directory src = NewDirectory();
+            DirectoryTaxonomyWriter w = new DirectoryTaxonomyWriter(src);
+            FacetLabel cp_b = new FacetLabel("b");
+            w.AddCategory(cp_b);
+            w.Dispose();
+
+            foreach (bool nrt in new bool[] { false, true })
+            {
+                Directory dir = NewDirectory();
+                var writer = new DirectoryTaxonomyWriter(dir);
+
+                FacetLabel cp_a = new FacetLabel("a");
+                writer.AddCategory(cp_a);
+                if (!nrt)
+                {
+                    writer.Commit();
+                }
+
+                DirectoryTaxonomyReader r1 = nrt ? new DirectoryTaxonomyReader(writer) : new DirectoryTaxonomyReader(dir);
+                // fill r1's caches
+                Assert.AreEqual(1, r1.GetOrdinal(cp_a));
+                Assert.AreEqual(cp_a, r1.GetPath(1));
+
+                // now replace taxonomy
+                writer.ReplaceTaxonomy(src);
+                if (!nrt)
+                {
+                    writer.Commit();
+                }
+
+                DirectoryTaxonomyReader r2 = TaxonomyReader.OpenIfChanged(r1);
+                Assert.NotNull(r2);
+
+                // fill r2's caches
+                Assert.AreEqual(1, r2.GetOrdinal(cp_b));
+                Assert.AreEqual(cp_b, r2.GetPath(1));
+
+                // check that r1 doesn't see cp_b
+                Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, r1.GetOrdinal(cp_b));
+                Assert.AreEqual(cp_a, r1.GetPath(1));
+
+                // check that r2 doesn't see cp_a
+                Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, r2.GetOrdinal(cp_a));
+                Assert.AreEqual(cp_b, r2.GetPath(1));
+
+                (r2).Dispose();
+                (r1).Dispose();
+                writer.Dispose();
+                dir.Dispose();
+            }
+
+            src.Dispose();
+        }
+
+        [Test]
+        public virtual void TestGetChildren()
+        {
+            Directory dir = NewDirectory();
+            var taxoWriter = new DirectoryTaxonomyWriter(dir);
+            int numCategories = AtLeast(10);
+            int numA = 0, numB = 0;
+            Random random = Random();
+            // add the two categories for which we'll also add children (so asserts are simpler)
+            taxoWriter.AddCategory(new FacetLabel("a"));
+            taxoWriter.AddCategory(new FacetLabel("b"));
+            for (int i = 0; i < numCategories; i++)
+            {
+                if (random.NextBoolean())
+                {
+                    taxoWriter.AddCategory(new FacetLabel("a", Convert.ToString(i)));
+                    ++numA;
+                }
+                else
+                {
+                    taxoWriter.AddCategory(new FacetLabel("b", Convert.ToString(i)));
+                    ++numB;
+                }
+            }
+            // add category with no children
+            taxoWriter.AddCategory(new FacetLabel("c"));
+            taxoWriter.Dispose();
+
+            var taxoReader = new DirectoryTaxonomyReader(dir);
+
+            // non existing category
+            TaxonomyReader.ChildrenIterator it = taxoReader.GetChildren(taxoReader.GetOrdinal(new FacetLabel("invalid")));
+            Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, it.Next());
+
+            // a category with no children
+            it = taxoReader.GetChildren(taxoReader.GetOrdinal(new FacetLabel("c")));
+            Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, it.Next());
+
+            // arbitrary negative ordinal
+            it = taxoReader.GetChildren(-2);
+            Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, it.Next());
+
+            // root's children
+            var roots = new HashSet<string>(Arrays.AsList("a", "b", "c"));
+            it = taxoReader.GetChildren(TaxonomyReader.ROOT_ORDINAL);
+            while (roots.Count > 0)
+            {
+                FacetLabel root = taxoReader.GetPath(it.Next());
+                Assert.AreEqual(1, root.Length);
+                Assert.True(roots.Remove(root.Components[0]));
+            }
+            Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, it.Next());
+
+            for (int i = 0; i < 2; i++)
+            {
+                FacetLabel cp = i == 0 ? new FacetLabel("a") : new FacetLabel("b");
+                int ordinal = taxoReader.GetOrdinal(cp);
+                it = taxoReader.GetChildren(ordinal);
+                int numChildren = 0;
+                int child;
+                while ((child = it.Next()) != TaxonomyReader.INVALID_ORDINAL)
+                {
+                    FacetLabel path = taxoReader.GetPath(child);
+                    Assert.AreEqual(2, path.Length);
+                    Assert.AreEqual(path.Components[0], i == 0 ? "a" : "b");
+                    ++numChildren;
+                }
+                int expected = i == 0 ? numA : numB;
+                Assert.AreEqual(expected, numChildren, "invalid num children");
+            }
+            taxoReader.Dispose();
+
+            dir.Dispose();
+        }
+
+    }
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/68aa9728/src/Lucene.Net.Tests.Facet/Taxonomy/Directory/TestDirectoryTaxonomyWriter.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Facet/Taxonomy/Directory/TestDirectoryTaxonomyWriter.cs b/src/Lucene.Net.Tests.Facet/Taxonomy/Directory/TestDirectoryTaxonomyWriter.cs
new file mode 100644
index 0000000..f5ddd11
--- /dev/null
+++ b/src/Lucene.Net.Tests.Facet/Taxonomy/Directory/TestDirectoryTaxonomyWriter.cs
@@ -0,0 +1,588 @@
+using System;
+using System.Collections.Generic;
+using System.Collections.Concurrent;
+using System.IO;
+using System.Threading;
+using Lucene.Net.Support;
+using NUnit.Framework;
+
+namespace Lucene.Net.Facet.Taxonomy.Directory
+{
+
+
+    using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer;
+    using Document = Lucene.Net.Documents.Document;
+    using MemoryOrdinalMap = Lucene.Net.Facet.Taxonomy.Directory.DirectoryTaxonomyWriter.MemoryOrdinalMap;
+    using TaxonomyWriterCache = Lucene.Net.Facet.Taxonomy.WriterCache.TaxonomyWriterCache;
+    using Cl2oTaxonomyWriterCache = Lucene.Net.Facet.Taxonomy.WriterCache.Cl2oTaxonomyWriterCache;
+    using LruTaxonomyWriterCache = Lucene.Net.Facet.Taxonomy.WriterCache.LruTaxonomyWriterCache;
+    using DirectoryReader = Lucene.Net.Index.DirectoryReader;
+    using IndexReader = Lucene.Net.Index.IndexReader;
+    using IndexWriter = Lucene.Net.Index.IndexWriter;
+    using OpenMode = Lucene.Net.Index.IndexWriterConfig.OpenMode_e;
+    using IndexWriterConfig = Lucene.Net.Index.IndexWriterConfig;
+    using SegmentInfos = Lucene.Net.Index.SegmentInfos;
+    using IndexSearcher = Lucene.Net.Search.IndexSearcher;
+    using AlreadyClosedException = Lucene.Net.Store.AlreadyClosedException;
+    using Directory = Lucene.Net.Store.Directory;
+    using IOUtils = Lucene.Net.Util.IOUtils;
+    using TestUtil = Lucene.Net.Util.TestUtil;
+
+    /*
+     * Licensed to the Apache Software Foundation (ASF) under one or more
+     * contributor license agreements.  See the NOTICE file distributed with
+     * this work for additional information regarding copyright ownership.
+     * The ASF licenses this file to You under the Apache License, Version 2.0
+     * (the "License"); you may not use this file except in compliance with
+     * the License.  You may obtain a copy of the License at
+     *
+     *     http://www.apache.org/licenses/LICENSE-2.0
+     *
+     * Unless required by applicable law or agreed to in writing, software
+     * distributed under the License is distributed on an "AS IS" BASIS,
+     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+     * See the License for the specific language governing permissions and
+     * limitations under the License.
+     */
+
+    public class TestDirectoryTaxonomyWriter : FacetTestCase
+    {
+
+        // A No-Op TaxonomyWriterCache which always discards all given categories, and
+        // always returns true in put(), to indicate some cache entries were cleared.
+        private static TaxonomyWriterCache NO_OP_CACHE = new TaxonomyWriterCacheAnonymousInnerClassHelper();
+
+        private class TaxonomyWriterCacheAnonymousInnerClassHelper : TaxonomyWriterCache
+        {
+            public TaxonomyWriterCacheAnonymousInnerClassHelper()
+            {
+            }
+
+
+            public virtual void Close()
+            {
+            }
+            public virtual int Get(FacetLabel categoryPath)
+            {
+                return -1;
+            }
+            public virtual bool Put(FacetLabel categoryPath, int ordinal)
+            {
+                return true;
+            }
+            public virtual bool Full
+            {
+                get
+                {
+                    return true;
+                }
+            }
+            public virtual void Clear()
+            {
+            }
+
+        }
+
+        [Test]
+        public virtual void TestCommit()
+        {
+            // Verifies that nothing is committed to the underlying Directory, if
+            // commit() wasn't called.
+            Directory dir = NewDirectory();
+            var ltw = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, NO_OP_CACHE);
+            Assert.False(DirectoryReader.IndexExists(dir));
+            ltw.Commit(); // first commit, so that an index will be created
+            ltw.AddCategory(new FacetLabel("a"));
+
+            IndexReader r = DirectoryReader.Open(dir);
+            Assert.AreEqual(1, r.NumDocs, "No categories should have been committed to the underlying directory");
+            r.Dispose();
+            ltw.Dispose();
+            dir.Dispose();
+        }
+
+        [Test]
+        public virtual void TestCommitUserData()
+        {
+            // Verifies taxonomy commit data
+            Directory dir = NewDirectory();
+            var taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, NO_OP_CACHE);
+            taxoWriter.AddCategory(new FacetLabel("a"));
+            taxoWriter.AddCategory(new FacetLabel("b"));
+            IDictionary<string, string> userCommitData = new Dictionary<string, string>();
+            userCommitData["testing"] = "1 2 3";
+            taxoWriter.CommitData = userCommitData;
+            taxoWriter.Dispose();
+            var r = DirectoryReader.Open(dir);
+            Assert.AreEqual(3, r.NumDocs, "2 categories plus root should have been committed to the underlying directory");
+            var readUserCommitData = r.IndexCommit.UserData;
+            Assert.True("1 2 3".Equals(readUserCommitData["testing"]), "wrong value extracted from commit data");
+            Assert.NotNull(DirectoryTaxonomyWriter.INDEX_EPOCH + " not found in commitData", readUserCommitData[DirectoryTaxonomyWriter.INDEX_EPOCH]);
+            r.Dispose();
+
+            // open DirTaxoWriter again and commit, INDEX_EPOCH should still exist
+            // in the commit data, otherwise DirTaxoReader.refresh() might not detect
+            // that the taxonomy index has been recreated.
+            taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, NO_OP_CACHE);
+            taxoWriter.AddCategory(new FacetLabel("c")); // add a category so that commit will happen
+
+
+            taxoWriter.CommitData = new Dictionary<string, string>()
+		    {
+			    {"just", "data"}
+		    };
+            taxoWriter.Commit();
+
+            // verify taxoWriter.getCommitData()
+            Assert.NotNull(DirectoryTaxonomyWriter.INDEX_EPOCH + " not found in taoxWriter.commitData", taxoWriter.CommitData[DirectoryTaxonomyWriter.INDEX_EPOCH]);
+            taxoWriter.Dispose();
+
+            r = DirectoryReader.Open(dir);
+            readUserCommitData = r.IndexCommit.UserData;
+            Assert.NotNull(DirectoryTaxonomyWriter.INDEX_EPOCH + " not found in commitData", readUserCommitData[DirectoryTaxonomyWriter.INDEX_EPOCH]);
+            r.Dispose();
+
+            dir.Dispose();
+        }
+
+        [Test]
+        public virtual void TestRollback()
+        {
+            // Verifies that if rollback is called, DTW is closed.
+            Directory dir = NewDirectory();
+            var dtw = new DirectoryTaxonomyWriter(dir);
+            dtw.AddCategory(new FacetLabel("a"));
+            dtw.Rollback();
+            try
+            {
+                dtw.AddCategory(new FacetLabel("a"));
+                Fail("should not have succeeded to add a category following rollback.");
+            }
+            catch (AlreadyClosedException)
+            {
+                // expected
+            }
+
+            dir.Dispose();
+        }
+
+        [Test]
+        public virtual void TestRecreateRollback()
+        {
+            // Tests rollback with OpenMode.CREATE
+            Directory dir = NewDirectory();
+            (new DirectoryTaxonomyWriter(dir)).Dispose();
+            Assert.AreEqual(1, getEpoch(dir));
+            (new DirectoryTaxonomyWriter(dir, OpenMode.CREATE)).Rollback();
+            Assert.AreEqual(1, getEpoch(dir));
+
+            dir.Dispose();
+        }
+
+        [Test]
+        public virtual void TestEnsureOpen()
+        {
+            // verifies that an exception is thrown if DTW was closed
+            Directory dir = NewDirectory();
+            DirectoryTaxonomyWriter dtw = new DirectoryTaxonomyWriter(dir);
+            dtw.Dispose();
+            try
+            {
+                dtw.AddCategory(new FacetLabel("a"));
+                Fail("should not have succeeded to add a category following close.");
+            }
+            catch (AlreadyClosedException)
+            {
+                // expected
+            }
+            dir.Dispose();
+        }
+
+        [Test]
+        private void TouchTaxo(DirectoryTaxonomyWriter taxoWriter, FacetLabel cp)
+        {
+            taxoWriter.AddCategory(cp);
+            taxoWriter.CommitData = new Dictionary<string, string>()
+		{
+			{"just", "data"}
+		};
+            taxoWriter.Commit();
+        }
+
+        [Test]
+        public virtual void TestRecreateAndRefresh()
+        {
+            // DirTaxoWriter lost the INDEX_EPOCH property if it was opened in
+            // CREATE_OR_APPEND (or commit(userData) called twice), which could lead to
+            // DirTaxoReader succeeding to refresh().
+            Directory dir = NewDirectory();
+
+            DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, NO_OP_CACHE);
+            TouchTaxo(taxoWriter, new FacetLabel("a"));
+
+            var taxoReader = new DirectoryTaxonomyReader(dir);
+
+            TouchTaxo(taxoWriter, new FacetLabel("b"));
+
+            var newtr = TaxonomyReader.OpenIfChanged(taxoReader);
+            taxoReader.Dispose();
+            taxoReader = newtr;
+            Assert.AreEqual(1, Convert.ToInt32(taxoReader.CommitUserData[DirectoryTaxonomyWriter.INDEX_EPOCH]));
+
+            // now recreate the taxonomy, and check that the epoch is preserved after opening DirTW again.
+            taxoWriter.Dispose();
+            taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE, NO_OP_CACHE);
+            TouchTaxo(taxoWriter, new FacetLabel("c"));
+            taxoWriter.Dispose();
+
+            taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, NO_OP_CACHE);
+            TouchTaxo(taxoWriter, new FacetLabel("d"));
+            taxoWriter.Dispose();
+
+            newtr = TaxonomyReader.OpenIfChanged(taxoReader);
+            taxoReader.Dispose();
+            taxoReader = newtr;
+            Assert.AreEqual(2, Convert.ToInt32(taxoReader.CommitUserData[DirectoryTaxonomyWriter.INDEX_EPOCH]));
+
+            taxoReader.Dispose();
+            dir.Dispose();
+        }
+
+        [Test]
+        public virtual void TestBackwardsCompatibility()
+        {
+            // tests that if the taxonomy index doesn't have the INDEX_EPOCH
+            // property (supports pre-3.6 indexes), all still works.
+            Directory dir = NewDirectory();
+
+            // create an empty index first, so that DirTaxoWriter initializes indexEpoch to 1.
+            (new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, null))).Dispose();
+
+            var taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, NO_OP_CACHE);
+            taxoWriter.Dispose();
+
+            var taxoReader = new DirectoryTaxonomyReader(dir);
+            Assert.AreEqual(1, Convert.ToInt32(taxoReader.CommitUserData[DirectoryTaxonomyWriter.INDEX_EPOCH]));
+            Assert.Null(TaxonomyReader.OpenIfChanged(taxoReader));
+            (taxoReader).Dispose();
+
+            dir.Dispose();
+        }
+
+        [Test]
+        public virtual void TestConcurrency()
+        {
+            int ncats = AtLeast(100000); // add many categories
+            int range = ncats * 3; // affects the categories selection
+            AtomicInteger numCats = new AtomicInteger(ncats);
+            Directory dir = NewDirectory();
+            var values = new ConcurrentDictionary<string, string>();
+            double d = Random().NextDouble();
+            TaxonomyWriterCache cache;
+            if (d < 0.7)
+            {
+                // this is the fastest, yet most memory consuming
+                cache = new Cl2oTaxonomyWriterCache(1024, 0.15f, 3);
+            }
+            else if (TEST_NIGHTLY && d > 0.98)
+            {
+                // this is the slowest, but tests the writer concurrency when no caching is done.
+                // only pick it during NIGHTLY tests, and even then, with very low chances.
+                cache = NO_OP_CACHE;
+            }
+            else
+            {
+                // this is slower than CL2O, but less memory consuming, and exercises finding categories on disk too.
+                cache = new LruTaxonomyWriterCache(ncats / 10);
+            }
+            if (VERBOSE)
+            {
+                Console.WriteLine("TEST: use cache=" + cache);
+            }
+            var tw = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE, cache);
+            ThreadClass[] addThreads = new ThreadClass[AtLeast(4)];
+            for (int z = 0; z < addThreads.Length; z++)
+            {
+                addThreads[z] = new ThreadAnonymousInnerClassHelper(this, range, numCats, values, tw);
+            }
+
+            foreach (var t in addThreads)
+            {
+                t.Start();
+            }
+            foreach (var t in addThreads)
+            {
+                t.Join();
+            }
+            tw.Dispose();
+
+            DirectoryTaxonomyReader dtr = new DirectoryTaxonomyReader(dir);
+            // +1 for root category
+            if (values.Count + 1 != dtr.Size)
+            {
+                foreach (string value in values.Keys)
+                {
+                    FacetLabel label = new FacetLabel(FacetsConfig.StringToPath(value));
+                    if (dtr.GetOrdinal(label) == -1)
+                    {
+                        Console.WriteLine("FAIL: path=" + label + " not recognized");
+                    }
+                }
+                Fail("mismatch number of categories");
+            }
+
+            int[] parents = dtr.ParallelTaxonomyArrays.Parents();
+            foreach (string cat in values.Keys)
+            {
+                FacetLabel cp = new FacetLabel(FacetsConfig.StringToPath(cat));
+                Assert.True(dtr.GetOrdinal(cp) > 0, "category not found " + cp);
+                int level = cp.Length;
+                int parentOrd = 0; // for root, parent is always virtual ROOT (ord=0)
+                FacetLabel path = new FacetLabel();
+                for (int i = 0; i < level; i++)
+                {
+                    path = cp.Subpath(i + 1);
+                    int ord = dtr.GetOrdinal(path);
+                    Assert.AreEqual(parentOrd, parents[ord], "invalid parent for cp=" + path);
+                    parentOrd = ord; // next level should have this parent
+                }
+            }
+
+            IOUtils.Close(dtr, dir);
+        }
+
+        private class ThreadAnonymousInnerClassHelper : ThreadClass
+        {
+            private readonly TestDirectoryTaxonomyWriter outerInstance;
+
+            private int range;
+            private AtomicInteger numCats;
+            private ConcurrentDictionary<string, string> values;
+            private Lucene.Net.Facet.Taxonomy.Directory.DirectoryTaxonomyWriter tw;
+
+            public ThreadAnonymousInnerClassHelper(TestDirectoryTaxonomyWriter outerInstance, int range, AtomicInteger numCats, ConcurrentDictionary<string, string> values, Lucene.Net.Facet.Taxonomy.Directory.DirectoryTaxonomyWriter tw)
+            {
+                this.outerInstance = outerInstance;
+                this.range = range;
+                this.numCats = numCats;
+                this.values = values;
+                this.tw = tw;
+            }
+
+            public override void Run()
+            {
+                Random random = Random();
+                while (numCats.DecrementAndGet() > 0)
+                {
+                    try
+                    {
+                        int value = random.Next(range);
+                        FacetLabel cp = new FacetLabel(Convert.ToString(value / 1000), Convert.ToString(value / 10000), Convert.ToString(value / 100000), Convert.ToString(value));
+                        int ord = tw.AddCategory(cp);
+                        Assert.True(tw.GetParent(ord) != -1, "invalid parent for ordinal " + ord + ", category " + cp);
+                        string l1 = FacetsConfig.PathToString(cp.Components, 1);
+                        string l2 = FacetsConfig.PathToString(cp.Components, 2);
+                        string l3 = FacetsConfig.PathToString(cp.Components, 3);
+                        string l4 = FacetsConfig.PathToString(cp.Components, 4);
+                        values[l1] = l1;
+                        values[l2] = l2;
+                        values[l3] = l3;
+                        values[l4] = l4;
+                    }
+                    catch (IOException e)
+                    {
+                        throw new Exception(e.Message, e);
+                    }
+                }
+            }
+        }
+
+        private long getEpoch(Directory taxoDir)
+        {
+            SegmentInfos infos = new SegmentInfos();
+            infos.Read(taxoDir);
+            return Convert.ToInt64(infos.UserData[DirectoryTaxonomyWriter.INDEX_EPOCH]);
+        }
+
+        [Test]
+        public virtual void TestReplaceTaxonomy()
+        {
+            Directory input = NewDirectory();
+            var taxoWriter = new DirectoryTaxonomyWriter(input);
+            int ordA = taxoWriter.AddCategory(new FacetLabel("a"));
+            taxoWriter.Dispose();
+
+            Directory dir = NewDirectory();
+            taxoWriter = new DirectoryTaxonomyWriter(dir);
+            int ordB = taxoWriter.AddCategory(new FacetLabel("b"));
+            taxoWriter.AddCategory(new FacetLabel("c"));
+            taxoWriter.Commit();
+
+            long origEpoch = getEpoch(dir);
+
+            // replace the taxonomy with the input one
+            taxoWriter.ReplaceTaxonomy(input);
+
+            // LUCENE-4633: make sure that category "a" is not added again in any case
+            taxoWriter.AddTaxonomy(input, new MemoryOrdinalMap());
+            Assert.AreEqual(2, taxoWriter.Size, "no categories should have been added"); // root + 'a'
+            Assert.AreEqual(ordA, taxoWriter.AddCategory(new FacetLabel("a")), "category 'a' received new ordinal?");
+
+            // add the same category again -- it should not receive the same ordinal !
+            int newOrdB = taxoWriter.AddCategory(new FacetLabel("b"));
+            Assert.AreNotSame(ordB, newOrdB, "new ordinal cannot be the original ordinal");
+            Assert.AreEqual(2, newOrdB, "ordinal should have been 2 since only one category was added by replaceTaxonomy");
+
+            taxoWriter.Dispose();
+
+            long newEpoch = getEpoch(dir);
+            Assert.True(origEpoch < newEpoch, "index epoch should have been updated after replaceTaxonomy");
+
+            dir.Dispose();
+            input.Dispose();
+        }
+
+        [Test]
+        public virtual void TestReaderFreshness()
+        {
+            // ensures that the internal index reader is always kept fresh. Previously,
+            // this simple scenario failed, if the cache just evicted the category that
+            // is being added.
+            Directory dir = NewDirectory();
+            DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE, NO_OP_CACHE);
+            int o1 = taxoWriter.AddCategory(new FacetLabel("a"));
+            int o2 = taxoWriter.AddCategory(new FacetLabel("a"));
+            Assert.True(o1 == o2, "ordinal for same category that is added twice should be the same !");
+            taxoWriter.Dispose();
+            dir.Dispose();
+        }
+
+        [Test]
+        public virtual void TestCommitNoEmptyCommits()
+        {
+            // LUCENE-4972: DTW used to create empty commits even if no changes were made
+            Directory dir = NewDirectory();
+            DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(dir);
+            taxoWriter.AddCategory(new FacetLabel("a"));
+            taxoWriter.Commit();
+
+            long gen1 = SegmentInfos.GetLastCommitGeneration(dir);
+            taxoWriter.Commit();
+            long gen2 = SegmentInfos.GetLastCommitGeneration(dir);
+            Assert.AreEqual(gen1, gen2, "empty commit should not have changed the index");
+
+            taxoWriter.Dispose();
+            dir.Dispose();
+        }
+
+        [Test]
+        public virtual void TestCloseNoEmptyCommits()
+        {
+            // LUCENE-4972: DTW used to create empty commits even if no changes were made
+            Directory dir = NewDirectory();
+            DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(dir);
+            taxoWriter.AddCategory(new FacetLabel("a"));
+            taxoWriter.Commit();
+
+            long gen1 = SegmentInfos.GetLastCommitGeneration(dir);
+            taxoWriter.Dispose();
+            long gen2 = SegmentInfos.GetLastCommitGeneration(dir);
+            Assert.AreEqual(gen1, gen2, "empty commit should not have changed the index");
+
+            taxoWriter.Dispose();
+            dir.Dispose();
+        }
+
+        [Test]
+        public virtual void TestPrepareCommitNoEmptyCommits()
+        {
+            // LUCENE-4972: DTW used to create empty commits even if no changes were made
+            Directory dir = NewDirectory();
+            DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(dir);
+            taxoWriter.AddCategory(new FacetLabel("a"));
+            taxoWriter.PrepareCommit();
+            taxoWriter.Commit();
+
+            long gen1 = SegmentInfos.GetLastCommitGeneration(dir);
+            taxoWriter.PrepareCommit();
+            taxoWriter.Commit();
+            long gen2 = SegmentInfos.GetLastCommitGeneration(dir);
+            Assert.AreEqual(gen1, gen2, "empty commit should not have changed the index");
+
+            taxoWriter.Dispose();
+            dir.Dispose();
+        }
+
+        [Test]
+        public virtual void TestHugeLabel()
+        {
+            Directory indexDir = NewDirectory(), taxoDir = NewDirectory();
+            IndexWriter indexWriter = new IndexWriter(indexDir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+            DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir, OpenMode.CREATE, new Cl2oTaxonomyWriterCache(2, 1f, 1));
+            FacetsConfig config = new FacetsConfig();
+
+            // Add one huge label:
+            string bigs = null;
+            int ordinal = -1;
+
+            int len = FacetLabel.MAX_CATEGORY_PATH_LENGTH - 4; // for the dimension and separator
+            bigs = TestUtil.RandomSimpleString(Random(), len, len);
+            FacetField ff = new FacetField("dim", bigs);
+            FacetLabel cp = new FacetLabel("dim", bigs);
+            ordinal = taxoWriter.AddCategory(cp);
+            Document doc = new Document();
+            doc.Add(ff);
+            indexWriter.AddDocument(config.Build(taxoWriter, doc));
+
+            // Add tiny ones to cause a re-hash
+            for (int i = 0; i < 3; i++)
+            {
+                string s = TestUtil.RandomSimpleString(Random(), 1, 10);
+                taxoWriter.AddCategory(new FacetLabel("dim", s));
+                doc = new Document();
+                doc.Add(new FacetField("dim", s));
+                indexWriter.AddDocument(config.Build(taxoWriter, doc));
+            }
+
+            // when too large components were allowed to be added, this resulted in a new added category
+            Assert.AreEqual(ordinal, taxoWriter.AddCategory(cp));
+
+            IOUtils.Close(indexWriter, taxoWriter);
+
+            DirectoryReader indexReader = DirectoryReader.Open(indexDir);
+            var taxoReader = new DirectoryTaxonomyReader(taxoDir);
+            IndexSearcher searcher = new IndexSearcher(indexReader);
+            DrillDownQuery ddq = new DrillDownQuery(new FacetsConfig());
+            ddq.Add("dim", bigs);
+            Assert.AreEqual(1, searcher.Search(ddq, 10).TotalHits);
+
+            IOUtils.Close(indexReader, taxoReader, indexDir, taxoDir);
+        }
+
+        [Test]
+        public virtual void TestReplaceTaxoWithLargeTaxonomy()
+        {
+            var srcTaxoDir = NewDirectory();
+            var targetTaxoDir = NewDirectory();
+
+            // build source, large, taxonomy
+            DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(srcTaxoDir);
+            int ord = taxoWriter.AddCategory(new FacetLabel("A", "1", "1", "1", "1", "1", "1"));
+            taxoWriter.Dispose();
+
+            taxoWriter = new DirectoryTaxonomyWriter(targetTaxoDir);
+            int ordinal = taxoWriter.AddCategory(new FacetLabel("B", "1"));
+            Assert.AreEqual(1, taxoWriter.GetParent(ordinal)); // call getParent to initialize taxoArrays
+            taxoWriter.Commit();
+
+            taxoWriter.ReplaceTaxonomy(srcTaxoDir);
+            Assert.AreEqual(ord - 1, taxoWriter.GetParent(ord));
+            taxoWriter.Dispose();
+
+            srcTaxoDir.Dispose();
+            targetTaxoDir.Dispose();
+        }
+
+    }
+
+}
\ No newline at end of file


Mime
View raw message