lucenenet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mhern...@apache.org
Subject [13/50] [abbrv] git commit: Port: more util test classes
Date Tue, 24 Sep 2013 18:32:49 GMT
Port: more util test classes


Project: http://git-wip-us.apache.org/repos/asf/lucenenet/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucenenet/commit/e02cc69c
Tree: http://git-wip-us.apache.org/repos/asf/lucenenet/tree/e02cc69c
Diff: http://git-wip-us.apache.org/repos/asf/lucenenet/diff/e02cc69c

Branch: refs/heads/branch_4x
Commit: e02cc69c378e5673f46eb2bc344ef0cec782aa03
Parents: b713f3b
Author: James Blair <jmblair7@gmail.com>
Authored: Wed Jul 17 18:17:49 2013 -0400
Committer: James Blair <jmblair7@gmail.com>
Committed: Wed Jul 17 18:17:49 2013 -0400

----------------------------------------------------------------------
 test/core/Lucene.Net.Test.csproj           |   5 +-
 test/core/Util/Cache/TestSimpleLRUCache.cs |  77 -----
 test/core/Util/Fst/Test2BFST.cs            | 335 +++++++++++++++++++
 test/core/Util/Fst/TestBytesStore.cs       | 408 ++++++++++++++++++++++++
 test/core/Util/TestFilterIterator.cs       |   5 +
 test/core/Util/TestFixedBitSet.cs          | 373 ++++++++++++++++++++++
 test/core/Util/TestIdentityHashSet.cs      |  45 +++
 7 files changed, 1169 insertions(+), 79 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucenenet/blob/e02cc69c/test/core/Lucene.Net.Test.csproj
----------------------------------------------------------------------
diff --git a/test/core/Lucene.Net.Test.csproj b/test/core/Lucene.Net.Test.csproj
index 2544dd6..cccd979 100644
--- a/test/core/Lucene.Net.Test.csproj
+++ b/test/core/Lucene.Net.Test.csproj
@@ -535,10 +535,12 @@
     <Compile Include="Util\Automaton\TestMinimize.cs" />
     <Compile Include="Util\Automaton\TestSpecialOperations.cs" />
     <Compile Include="Util\Automaton\TestUTF32ToUTF8.cs" />
-    <Compile Include="Util\Cache\TestSimpleLRUCache.cs" />
     <Compile Include="Util\English.cs">
       <SubType>Code</SubType>
     </Compile>
+    <Compile Include="Util\Fst\Test2BFST.cs" />
+    <Compile Include="Util\Fst\TestBytesStore.cs" />
+    <Compile Include="Util\Fst\TestFSTs.cs" />
     <Compile Include="Util\LocalizedTestCase.cs" />
     <Compile Include="Util\Paths.cs" />
     <Compile Include="Util\StressRamUsageEstimator.cs" />
@@ -648,7 +650,6 @@
     <Content Include="UpdatedTests.txt" />
   </ItemGroup>
   <ItemGroup>
-    <Folder Include="Util\Fst\" />
     <Folder Include="Util\Packed\" />
   </ItemGroup>
   <Import Project="$(MSBuildBinPath)\Microsoft.CSharp.targets" />

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/e02cc69c/test/core/Util/Cache/TestSimpleLRUCache.cs
----------------------------------------------------------------------
diff --git a/test/core/Util/Cache/TestSimpleLRUCache.cs b/test/core/Util/Cache/TestSimpleLRUCache.cs
deleted file mode 100644
index a33c2c7..0000000
--- a/test/core/Util/Cache/TestSimpleLRUCache.cs
+++ /dev/null
@@ -1,77 +0,0 @@
-/* 
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-using System;
-
-using NUnit.Framework;
-
-using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
-
-namespace Lucene.Net.Util.Cache
-{
-	
-	[TestFixture]
-	public class TestSimpleLRUCache:LuceneTestCase
-	{
-		
-		[Test]
-		public virtual void  TestLRUCache()
-		{
-			int n = 100;
-            object dummy = new object();
-
-            Cache<int, object> cache = new SimpleLRUCache<int, object>(n);
-			
-			for (int i = 0; i < n; i++)
-			{
-				cache.Put(i, dummy);
-			}
-			
-			// access every 2nd item in cache
-			for (int i = 0; i < n; i += 2)
-			{
-				Assert.IsNotNull(cache.Get(i));
-			}
-			
-			// add n/2 elements to cache, the ones that weren't
-			// touched in the previous loop should now be thrown away
-			for (int i = n; i < n + (n / 2); i++)
-			{
-				cache.Put(i, dummy);
-			}
-			
-			// access every 4th item in cache
-			for (int i = 0; i < n; i += 4)
-			{
-				Assert.IsNotNull(cache.Get(i));
-			}
-			
-			// add 3/4n elements to cache, the ones that weren't
-			// touched in the previous loops should now be thrown away
-			for (int i = n; i < n + (n * 3 / 4); i++)
-			{
-				cache.Put(i, dummy);
-			}
-			
-			// access every 4th item in cache
-			for (int i = 0; i < n; i += 4)
-			{
-				Assert.IsNotNull(cache.Get(i));
-			}
-		}
-	}
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/e02cc69c/test/core/Util/Fst/Test2BFST.cs
----------------------------------------------------------------------
diff --git a/test/core/Util/Fst/Test2BFST.cs b/test/core/Util/Fst/Test2BFST.cs
new file mode 100644
index 0000000..857f912
--- /dev/null
+++ b/test/core/Util/Fst/Test2BFST.cs
@@ -0,0 +1,335 @@
+using System;
+using Lucene.Net.Store;
+using Lucene.Net.Support;
+using Lucene.Net.Test.Support;
+using Lucene.Net.Util;
+using Lucene.Net.Util.Fst;
+using Lucene.Net.Util.Packed;
+using NUnit.Framework;
+
+namespace Lucene.Net.Test.Util.Fst
+{
+    [TestFixture]
+    [Ignore("Requires tons of heap to run (10G works)")]
+    [Timeout(360000000)] // @TimeoutSuite(millis = 100 * TimeUnits.HOUR)
+    public class Test2BFST : LuceneTestCase
+    {
+        private static long LIMIT = 3L * 1024 * 1024 * 1024;
+
+        [Test]
+        public void Test()
+        {
+            var ints = new int[7];
+            var input = new IntsRef(ints, 0, ints.Length);
+            var seed = new Random().NextLong();
+
+            Directory dir = new MMapDirectory(_TestUtil.GetTempDir("2BFST"));
+
+            for (var doPackIter = 0; doPackIter < 2; doPackIter++)
+            {
+                var doPack = doPackIter == 1;
+
+                // Build FST w/ NoOutputs and stop when nodeCount > 3B
+                if (!doPack)
+                {
+                    Console.WriteLine("\nTEST: 3B nodes; doPack=false output=NO_OUTPUTS");
+                    Outputs<object> outputs = NoOutputs.GetSingleton();
+                    var NO_OUTPUT = outputs.GetNoOutput();
+                    var b = new Builder<object>(FST.INPUT_TYPE.BYTE1, 0, 0, false, false, int.MaxValue, outputs,
+                                                                  null, doPack, PackedInts.COMPACT, true, 15);
+
+                    var count = 0;
+                    var r = new Random((int)seed);
+                    var ints2 = new int[200];
+                    var input2 = new IntsRefs(ints2, 0, ints2.Length);
+                    while (true)
+                    {
+                        //Console.WriteLine("add: " + input + " -> " + output);
+                        for (var i = 10; i < ints2.Length; i++)
+                        {
+                            ints2[i] = r.Next(256);
+                        }
+                        b.Add(input2, NO_OUTPUT);
+                        count++;
+                        if (count % 100000 == 0)
+                        {
+                            Console.WriteLine(count + ": " + b.FstSizeInBytes() + " bytes; " + b.TotStateCount + " nodes");
+                        }
+                        if (b.TotStateCount > LIMIT)
+                        {
+                            break;
+                        }
+                        NextInput(r, ints2);
+                    }
+
+                    var fst = b.Finish();
+
+                    for (var verify = 0; verify < 2; verify++)
+                    {
+                        Console.WriteLine("\nTEST: now verify [fst size=" + fst.SizeInBytes() + "; nodeCount=" + fst.NodeCount + "; arcCount=" + fst.ArcCount + "]");
+
+                        Arrays.Fill(ints2, 0);
+                        r = new Random((int)seed);
+
+                        for (var i = 0; i < count; i++)
+                        {
+                            if (i % 1000000 == 0)
+                            {
+                                Console.WriteLine(i + "...: ");
+                            }
+                            for (int j = 10; j < ints2.Length; j++)
+                            {
+                                ints2[j] = r.Next(256);
+                            }
+                            assertEquals(NO_OUTPUT, Util.get(fst, input2));
+                            NextInput(r, ints2);
+                        }
+
+                        Console.WriteLine("\nTEST: enum all input/outputs");
+                        var fstEnum = new IntsRefFSTEnum<object>(fst);
+
+                        Arrays.Fill(ints2, 0);
+                        r = new Random((int)seed);
+                        var upto = 0;
+                        while (true)
+                        {
+                            var pair = fstEnum.Next();
+                            if (pair == null)
+                            {
+                                break;
+                            }
+                            for (int j = 10; j < ints2.Length; j++)
+                            {
+                                ints2[j] = r.Next(256);
+                            }
+                            assertEquals(input2, pair.Input);
+                            assertEquals(NO_OUTPUT, pair.Output);
+                            upto++;
+                            NextInput(r, ints2);
+                        }
+                        assertEquals(count, upto);
+
+                        if (verify == 0)
+                        {
+                            Console.WriteLine("\nTEST: save/load FST and re-verify");
+                            var output = dir.CreateOutput("fst", IOContext.DEFAULT);
+                            fst.Save(output);
+                            output.Dispose();
+                            var input3 = dir.OpenInput("fst", IOContext.DEFAULT);
+                            fst = new FST<object>(input3, outputs);
+                            input3.Dispose();
+                        }
+                        else
+                        {
+                            dir.DeleteFile("fst");
+                        }
+                    }
+                }
+
+                // Build FST w/ ByteSequenceOutputs and stop when FST
+                // size = 3GB
+                {
+                    Console.WriteLine("\nTEST: 3 GB size; doPack=" + doPack + " outputs=bytes");
+                    Outputs<BytesRef> outputs = ByteSequenceOutputs.GetSingleton();
+                    var b = new Builder<BytesRef>(FST.INPUT_TYPE.BYTE1, 0, 0, true, true, int.MaxValue, outputs,
+                                                                      null, doPack, PackedInts.COMPACT, true, 15);
+
+                    var outputBytes = new byte[20];
+                    var output = new BytesRef(outputBytes);
+                    Arrays.Fill(ints, 0);
+                    var count = 0;
+                    var r = new Random(seed);
+                    while (true)
+                    {
+                        r.NextBytes(outputBytes);
+                        //Console.WriteLine("add: " + input + " -> " + output);
+                        b.Add(input, BytesRef.DeepCopyOf(output));
+                        count++;
+                        if (count % 1000000 == 0)
+                        {
+                            Console.WriteLine(count + "...: " + b.FstSizeInBytes() + " bytes");
+                        }
+                        if (b.FstSizeInBytes() > LIMIT)
+                        {
+                            break;
+                        }
+                        NextInput(r, ints);
+                    }
+
+                    FST<BytesRef> fst = b.Finish();
+                    for (int verify = 0; verify < 2; verify++)
+                    {
+
+                        Console.WriteLine("\nTEST: now verify [fst size=" + fst.SizeInBytes() + "; nodeCount=" + fst.NodeCount + "; arcCount=" + fst.ArcCount + "]");
+
+                        r = new Random(seed);
+                        Arrays.Fill(ints, 0);
+
+                        for (int i = 0; i < count; i++)
+                        {
+                            if (i % 1000000 == 0)
+                            {
+                                Console.WriteLine(i + "...: ");
+                            }
+                            r.NextBytes(outputBytes);
+                            assertEquals(output, Util.get(fst, input));
+                            NextInput(r, ints);
+                        }
+
+                        Console.WriteLine("\nTEST: enum all input/outputs");
+                        var fstEnum = new IntsRefFSTEnum<BytesRef>(fst);
+
+                        Arrays.Fill(ints, 0);
+                        r = new Random((int)seed);
+                        int upto = 0;
+                        while (true)
+                        {
+                            var pair = fstEnum.Next();
+                            if (pair == null)
+                            {
+                                break;
+                            }
+                            assertEquals(input, pair.Input);
+                            r.NextBytes(outputBytes);
+                            assertEquals(output, pair.Output);
+                            upto++;
+                            NextInput(r, ints);
+                        }
+                        assertEquals(count, upto);
+
+                        if (verify == 0)
+                        {
+                            Console.WriteLine("\nTEST: save/load FST and re-verify");
+                            var output2 = dir.CreateOutput("fst", IOContext.DEFAULT);
+                            fst.Save(output2);
+                            output2.Dispose();
+                            var input4 = dir.OpenInput("fst", IOContext.DEFAULT);
+                            fst = new FST<BytesRef>(input4, outputs);
+                            input4.Dispose();
+                        }
+                        else
+                        {
+                            dir.DeleteFile("fst");
+                        }
+                    }
+                }
+
+                // Build FST w/ PositiveIntOutputs and stop when FST
+                // size = 3GB
+                {
+                    Console.WriteLine("\nTEST: 3 GB size; doPack=" + doPack + " outputs=long");
+                    Outputs<long> outputs = PositiveIntOutputs.GetSingleton();
+                    Builder<long> b = new Builder<long>(FST.INPUT_TYPE.BYTE1, 0, 0, true, true, int.MaxValue, outputs,
+                                                              null, doPack, PackedInts.COMPACT, true, 15);
+
+                    long output = 1;
+
+                    Arrays.Fill(ints, 0);
+                    var count = 0;
+                    var r = new Random(seed);
+                    while (true)
+                    {
+                        //Console.WriteLine("add: " + input + " -> " + output);
+                        b.Add(input, output);
+                        output += 1 + r.Next(10);
+                        count++;
+                        if (count % 1000000 == 0)
+                        {
+                            Console.WriteLine(count + "...: " + b.FstSizeInBytes() + " bytes");
+                        }
+                        if (b.FstSizeInBytes() > LIMIT)
+                        {
+                            break;
+                        }
+                        NextInput(r, ints);
+                    }
+
+                    FST<long> fst = b.Finish();
+
+                    for (int verify = 0; verify < 2; verify++)
+                    {
+
+                        Console.WriteLine("\nTEST: now verify [fst size=" + fst.SizeInBytes() + "; nodeCount=" + fst.NodeCount + "; arcCount=" + fst.ArcCount + "]");
+
+                        Arrays.Fill(ints, 0);
+
+                        output = 1;
+                        r = new Random(seed);
+                        for (int i = 0; i < count; i++)
+                        {
+                            if (i % 1000000 == 0)
+                            {
+                                Console.WriteLine(i + "...: ");
+                            }
+
+                            // forward lookup:
+                            assertEquals(output, Util.get(fst, input).longValue());
+                            // reverse lookup:
+                            assertEquals(input, Util.getByOutput(fst, output));
+                            output += 1 + r.Next(10);
+                            NextInput(r, ints);
+                        }
+
+                        Console.WriteLine("\nTEST: enum all input/outputs");
+                        IntsRefFSTEnum<long> fstEnum = new IntsRefFSTEnum<long>(fst);
+
+                        Arrays.Fill(ints, 0);
+                        r = new Random((int)seed);
+                        int upto = 0;
+                        output = 1;
+                        while (true)
+                        {
+                            var pair = fstEnum.Next();
+                            if (pair == null)
+                            {
+                                break;
+                            }
+                            assertEquals(input, pair.Input);
+                            assertEquals(output, pair.Output);
+                            output += 1 + r.Next(10);
+                            upto++;
+                            NextInput(r, ints);
+                        }
+                        assertEquals(count, upto);
+
+                        if (verify == 0)
+                        {
+                            Console.WriteLine("\nTEST: save/load FST and re-verify");
+                            var output3 = dir.CreateOutput("fst", IOContext.DEFAULT);
+                            fst.Save(output3);
+                            output3.Dispose();
+                            var input5 = dir.OpenInput("fst", IOContext.DEFAULT);
+                            fst = new FST<long>(input5, outputs);
+                            input5.Dispose();
+                        }
+                        else
+                        {
+                            dir.DeleteFile("fst");
+                        }
+                    }
+                }
+            }
+            dir.Dispose();
+        }
+
+        private void NextInput(Random r, int[] ints)
+        {
+            var downTo = 6;
+            while (downTo >= 0)
+            {
+                // Must add random amounts (and not just 1) because
+                // otherwise FST outsmarts us and remains tiny:
+                ints[downTo] += 1 + r.Next(10);
+                if (ints[downTo] < 256)
+                {
+                    break;
+                }
+                else
+                {
+                    ints[downTo] = 0;
+                    downTo--;
+                }
+            }
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/e02cc69c/test/core/Util/Fst/TestBytesStore.cs
----------------------------------------------------------------------
diff --git a/test/core/Util/Fst/TestBytesStore.cs b/test/core/Util/Fst/TestBytesStore.cs
new file mode 100644
index 0000000..df435a2
--- /dev/null
+++ b/test/core/Util/Fst/TestBytesStore.cs
@@ -0,0 +1,408 @@
+using System;
+using Lucene.Net.Store;
+using Lucene.Net.Support;
+using Lucene.Net.Test.Support;
+using Lucene.Net.Util;
+using Lucene.Net.Util.Fst;
+using NUnit.Framework;
+
+namespace Lucene.Net.Test.Util.Fst
+{
+    [TestFixture]
+    public class TestBytesStore : LuceneTestCase
+    {
+        [Test]
+        public void TestRandom()
+        {
+
+            int iters = AtLeast(10);
+            for (var iter = 0; iter < iters; iter++)
+            {
+                int numBytes = _TestUtil.NextInt(new Random(), 1, 200000);
+                var expected = new sbyte[numBytes];
+                int blockBits = _TestUtil.NextInt(new Random(), 8, 15);
+                var bytes = new BytesStore(blockBits);
+                if (VERBOSE)
+                {
+                    Console.WriteLine("TEST: iter=" + iter + " numBytes=" + numBytes + " blockBits=" + blockBits);
+                }
+
+                var pos = 0;
+                while (pos < numBytes)
+                {
+                    int op = new Random().Next(8);
+                    if (VERBOSE)
+                    {
+                        Console.WriteLine("  cycle pos=" + pos);
+                    }
+                    switch (op)
+                    {
+
+                        case 0:
+                            {
+                                // write random byte
+                                var b = (sbyte)new Random().Next(256);
+                                if (VERBOSE)
+                                {
+                                    Console.WriteLine("    writeByte b=" + b);
+                                }
+
+                                expected[pos++] = b;
+                                bytes.WriteByte(b);
+                            }
+                            break;
+
+                        case 1:
+                            {
+                                // write random byte[]
+                                var len = new Random().Next(Math.Min(numBytes - pos, 100));
+                                var temp = new byte[len];
+                                new Random().NextBytes(temp);
+                                if (VERBOSE)
+                                {
+                                    Console.WriteLine("    writeBytes len=" + len + " bytes=" + Arrays.ToString(temp));
+                                }
+                                Array.Copy(temp, 0, expected, pos, temp.Length);
+                                bytes.WriteBytes(temp, 0, temp.Length);
+                                pos += len;
+                            }
+                            break;
+
+                        case 2:
+                            {
+                                // write int @ absolute pos
+                                if (pos > 4)
+                                {
+                                    int x = new Random().Next();
+                                    int randomPos = new Random().Next(pos - 4);
+                                    if (VERBOSE)
+                                    {
+                                        Console.WriteLine("    abs writeInt pos=" + randomPos + " x=" + x);
+                                    }
+                                    bytes.WriteInt(randomPos, x);
+                                    expected[randomPos++] = (sbyte)(x >> 24);
+                                    expected[randomPos++] = (sbyte)(x >> 16);
+                                    expected[randomPos++] = (sbyte)(x >> 8);
+                                    expected[randomPos++] = (sbyte)x;
+                                }
+                            }
+                            break;
+
+                        case 3:
+                            {
+                                // reverse bytes
+                                if (pos > 1)
+                                {
+                                    int len = _TestUtil.NextInt(new Random(), 2, Math.Min(100, pos));
+                                    int start;
+                                    if (len == pos)
+                                    {
+                                        start = 0;
+                                    }
+                                    else
+                                    {
+                                        start = new Random().Next(pos - len);
+                                    }
+                                    var end = start + len - 1;
+                                    if (VERBOSE)
+                                    {
+                                        Console.WriteLine("    reverse start=" + start + " end=" + end + " len=" + len + " pos=" + pos);
+                                    }
+                                    bytes.Reverse(start, end);
+
+                                    while (start <= end)
+                                    {
+                                        var b = expected[end];
+                                        expected[end] = expected[start];
+                                        expected[start] = b;
+                                        start++;
+                                        end--;
+                                    }
+                                }
+                            }
+                            break;
+
+                        case 4:
+                            {
+                                // abs write random byte[]
+                                if (pos > 2)
+                                {
+                                    int randomPos = new Random().Next(pos - 1);
+                                    int len = _TestUtil.NextInt(new Random(), 1, Math.Min(pos - randomPos - 1, 100));
+                                    byte[] temp = new byte[len];
+                                    new Random().NextBytes(temp);
+                                    if (VERBOSE)
+                                    {
+                                        Console.WriteLine("    abs writeBytes pos=" + randomPos + " len=" + len + " bytes=" + Arrays.ToString(temp));
+                                    }
+                                    Array.Copy(temp, 0, expected, randomPos, temp.Length);
+                                    bytes.WriteBytes(randomPos, temp, 0, temp.Length);
+                                }
+                            }
+                            break;
+
+                        case 5:
+                            {
+                                // copyBytes
+                                if (pos > 1)
+                                {
+                                    int src = new Random().Next(pos - 1);
+                                    int dest = _TestUtil.NextInt(new Random(), src + 1, pos - 1);
+                                    int len = _TestUtil.NextInt(new Random(), 1, Math.Min(300, pos - dest));
+                                    if (VERBOSE)
+                                    {
+                                        Console.WriteLine("    copyBytes src=" + src + " dest=" + dest + " len=" + len);
+                                    }
+                                    Array.Copy(expected, src, expected, dest, len);
+                                    bytes.CopyBytes(src, dest, len);
+                                }
+                            }
+                            break;
+
+                        case 6:
+                            {
+                                // skip
+                                var len = new Random().Next(Math.Min(100, numBytes - pos));
+
+                                if (VERBOSE)
+                                {
+                                    Console.WriteLine("    skip len=" + len);
+                                }
+
+                                pos += len;
+                                bytes.SkipBytes(len);
+
+                                // NOTE: must fill in zeros in case truncate was
+                                // used, else we get false fails:
+                                if (len > 0)
+                                {
+                                    var zeros = new sbyte[len];
+                                    bytes.WriteBytes(pos - len, zeros, 0, len);
+                                }
+                            }
+                            break;
+
+                        case 7:
+                            {
+                                // absWriteByte
+                                if (pos > 0)
+                                {
+                                    var dest = new Random().Next(pos);
+                                    var b = (sbyte)new Random().Next(256);
+                                    expected[dest] = b;
+                                    bytes.WriteByte(dest, b);
+                                }
+                                break;
+                            }
+                    }
+
+                    Assert.AreEqual(pos, bytes.GetPosition());
+
+                    if (pos > 0 && new Random().Next(50) == 17)
+                    {
+                        // truncate
+                        int len = _TestUtil.NextInt(new Random(), 1, Math.Min(pos, 100));
+                        bytes.Truncate(pos - len);
+                        pos -= len;
+                        Arrays.Fill(expected, pos, pos + len, (sbyte)0);
+                        if (VERBOSE)
+                        {
+                            Console.WriteLine("    truncate len=" + len + " newPos=" + pos);
+                        }
+                    }
+
+                    if ((pos > 0 && new Random().Next(200) == 17))
+                    {
+                        Verify(bytes, expected, pos);
+                    }
+                }
+
+                BytesStore bytesToVerify;
+
+                if (new Random().NextBool())
+                {
+                    if (VERBOSE)
+                    {
+                        Console.WriteLine("TEST: save/load  bytes");
+                    }
+                    Directory dir = NewDirectory();
+                    var output = dir.CreateOutput("bytes", IOContext.DEFAULT);
+                    bytes.WriteTo(output);
+                    output.Dispose();
+                    var input = dir.OpenInput("bytes", IOContext.DEFAULT);
+                    bytesToVerify = new BytesStore(input, numBytes, _TestUtil.NextInt(new Random(), 256, int.MaxValue));
+                    input.Dispose();
+                    dir.Dispose();
+                }
+                else
+                {
+                    bytesToVerify = bytes;
+                }
+
+                Verify(bytesToVerify, expected, numBytes);
+            }
+        }
+
+        private void Verify(BytesStore bytes, sbyte[] expected, int totalLength)
+        {
+            Assert.AreEqual(totalLength, bytes.GetPosition());
+            if (totalLength == 0)
+            {
+                return;
+            }
+            if (VERBOSE)
+            {
+                Console.WriteLine("  verify...");
+            }
+
+            // First verify whole thing in one blast:
+            var actual = new sbyte[totalLength];
+            if (new Random().NextBool())
+            {
+                if (VERBOSE)
+                {
+                    Console.WriteLine("    bulk: reversed");
+                }
+                // reversed
+                var reverseReader = bytes.GetReverseReader();
+                Assert.IsTrue(reverseReader.Reversed());
+                reverseReader.Position = totalLength - 1;
+                reverseReader.ReadBytes(actual, 0, actual.Length);
+                var start = 0;
+                var end = totalLength - 1;
+                while (start < end)
+                {
+                    var b = actual[start];
+                    actual[start] = actual[end];
+                    actual[end] = b;
+                    start++;
+                    end--;
+                }
+            }
+            else
+            {
+                // forward
+                if (VERBOSE)
+                {
+                    Console.WriteLine("    bulk: forward");
+                }
+                var forwardReader = bytes.GetForwardReader();
+                Assert.IsFalse(forwardReader.Reversed());
+                forwardReader.ReadBytes(actual, 0, actual.Length);
+            }
+
+            for (int i = 0; i < totalLength; i++)
+            {
+                Assert.AreEqual(expected[i], actual[i], "byte @ index=" + i);
+            }
+
+            FST.BytesReader r;
+
+            // Then verify ops:
+            bool reversed = new Random().NextBool();
+            if (reversed)
+            {
+                if (VERBOSE)
+                {
+                    Console.WriteLine("    ops: reversed");
+                }
+                r = bytes.GetReverseReader();
+            }
+            else
+            {
+                if (VERBOSE)
+                {
+                    Console.WriteLine("    ops: forward");
+                }
+                r = bytes.GetForwardReader();
+            }
+
+            if (totalLength > 1)
+            {
+                int numOps = _TestUtil.NextInt(new Random(), 100, 200);
+                for (int op = 0; op < numOps; op++)
+                {
+
+                    int numBytes = new Random().Next(Math.Min(1000, totalLength - 1));
+                    int pos;
+                    if (reversed)
+                    {
+                        pos = _TestUtil.NextInt(new Random(), numBytes, totalLength - 1);
+                    }
+                    else
+                    {
+                        pos = new Random().Next(totalLength - numBytes);
+                    }
+                    if (VERBOSE)
+                    {
+                        Console.WriteLine("    op iter=" + op + " reversed=" + reversed + " numBytes=" + numBytes + " pos=" + pos);
+                    }
+                    var temp = new sbyte[numBytes];
+                    r.Position = pos;
+                    Assert.AreEqual(pos, r.Position);
+                    r.ReadBytes(temp, 0, temp.Length);
+                    for (int i = 0; i < numBytes; i++)
+                    {
+                        sbyte expectedByte;
+                        if (reversed)
+                        {
+                            expectedByte = expected[pos - i];
+                        }
+                        else
+                        {
+                            expectedByte = expected[pos + i];
+                        }
+                        Assert.AreEqual(expectedByte, temp[i], "byte @ index=" + i);
+                    }
+
+                    int left;
+                    int expectedPos;
+
+                    if (reversed)
+                    {
+                        expectedPos = pos - numBytes;
+                        left = (int)r.Position;
+                    }
+                    else
+                    {
+                        expectedPos = pos + numBytes;
+                        left = (int)(totalLength - r.Position);
+                    }
+                    Assert.AreEqual(expectedPos, r.Position);
+
+                    if (left > 4)
+                    {
+                        int skipBytes = new Random().Next(left - 4);
+
+                        int expectedInt = 0;
+                        if (reversed)
+                        {
+                            expectedPos -= skipBytes;
+                            expectedInt |= (expected[expectedPos--] & 0xFF) << 24;
+                            expectedInt |= (expected[expectedPos--] & 0xFF) << 16;
+                            expectedInt |= (expected[expectedPos--] & 0xFF) << 8;
+                            expectedInt |= (expected[expectedPos--] & 0xFF);
+                        }
+                        else
+                        {
+                            expectedPos += skipBytes;
+                            expectedInt |= (expected[expectedPos++] & 0xFF) << 24;
+                            expectedInt |= (expected[expectedPos++] & 0xFF) << 16;
+                            expectedInt |= (expected[expectedPos++] & 0xFF) << 8;
+                            expectedInt |= (expected[expectedPos++] & 0xFF);
+                        }
+
+                        if (VERBOSE)
+                        {
+                            Console.WriteLine("    skip numBytes=" + skipBytes);
+                            Console.WriteLine("    readInt");
+                        }
+
+                        r.SkipBytes(skipBytes);
+                        Assert.AreEqual(expectedInt, r.ReadInt());
+                    }
+                }
+            }
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/e02cc69c/test/core/Util/TestFilterIterator.cs
----------------------------------------------------------------------
diff --git a/test/core/Util/TestFilterIterator.cs b/test/core/Util/TestFilterIterator.cs
new file mode 100644
index 0000000..04816d4
--- /dev/null
+++ b/test/core/Util/TestFilterIterator.cs
@@ -0,0 +1,5 @@
+namespace Lucene.Net.Test.Util
+{
+    // There is no FilterIterator class, as LINQ handles this very well with the Where method.
+    // This file is here to help with matching up files with the java file structure.
+}

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/e02cc69c/test/core/Util/TestFixedBitSet.cs
----------------------------------------------------------------------
diff --git a/test/core/Util/TestFixedBitSet.cs b/test/core/Util/TestFixedBitSet.cs
new file mode 100644
index 0000000..82d3e3c
--- /dev/null
+++ b/test/core/Util/TestFixedBitSet.cs
@@ -0,0 +1,373 @@
+using System;
+using System.Collections;
+using Lucene.Net.Search;
+using Lucene.Net.Support;
+using Lucene.Net.Test.Support;
+using Lucene.Net.Util;
+using NUnit.Framework;
+
+namespace Lucene.Net.Test.Util
+{
+    [TestFixture]
+    public class TestFixedBitSet : LuceneTestCase
+    {
+        internal virtual void DoGet(BitArray a, FixedBitSet b)
+        {
+            int max = b.Length;
+            for (var i = 0; i < max; i++)
+            {
+                if (a[i] != b[i])
+                {
+                    Fail("mismatch: BitArray=[" + i + "]=" + a[i]);
+                }
+            }
+        }
+
+        internal virtual void doNextSetBit(BitArray a, FixedBitSet b)
+        {
+            int aa = -1, bb = -1;
+            do
+            {
+                aa = a.NextSetBit(aa + 1);
+                bb = bb < b.Length - 1 ? b.NextSetBit(bb + 1) : -1;
+                Assert.Equals(aa, bb);
+            } while (aa >= 0);
+        }
+
+        internal virtual void doPrevSetBit(BitArray a, FixedBitSet b)
+        {
+            int aa = a.Length + new Random().Next(100);
+            int bb = aa;
+            do
+            {
+                // aa = a.prevSetBit(aa-1);
+                aa--;
+                while ((aa >= 0) && (!a[aa]))
+                {
+                    aa--;
+                }
+                if (b.Length == 0)
+                {
+                    bb = -1;
+                }
+                else if (bb > b.Length - 1)
+                {
+                    bb = b.PrevSetBit(b.Length - 1);
+                }
+                else if (bb < 1)
+                {
+                    bb = -1;
+                }
+                else
+                {
+                    bb = bb >= 1 ? b.PrevSetBit(bb - 1) : -1;
+                }
+                Assert.Equals(aa, bb);
+            } while (aa >= 0);
+        }
+
+        // test interleaving different FixedBitSetIterator.next()/skipTo()
+        internal virtual void DoIterate(BitArray a, FixedBitSet b, int mode)
+        {
+            if (mode == 1) DoIterate1(a, b);
+            if (mode == 2) DoIterate2(a, b);
+        }
+
+        internal virtual void DoIterate1(BitArray a, FixedBitSet b)
+        {
+            int aa = -1, bb = -1;
+            var iterator = b.Iterator();
+            do
+            {
+                aa = a.NextSetBit(aa + 1);
+                bb = (bb < b.Length && new Random().NextBool()) ? iterator.NextDoc() : iterator.Advance(bb + 1);
+                Assert.Equals(aa == -1 ? DocIdSetIterator.NO_MORE_DOCS : aa, bb);
+            } while (aa >= 0);
+        }
+
+        internal virtual void DoIterate2(BitArray a, FixedBitSet b)
+        {
+            int aa = -1, bb = -1;
+            var iterator = b.Iterator();
+            do
+            {
+                aa = a.NextSetBit(aa + 1);
+                bb = new Random().NextBool() ? iterator.NextDoc() : iterator.Advance(bb + 1);
+                Assert.Equals(aa == -1 ? DocIdSetIterator.NO_MORE_DOCS : aa, bb);
+            } while (aa >= 0);
+        }
+
+        internal virtual void DoRandomSets(int maxSize, int iter, int mode)
+        {
+            BitArray a0 = null;
+            FixedBitSet b0 = null;
+
+            var random = new Random();
+
+            for (var i = 0; i < iter; i++)
+            {
+                int sz = _TestUtil.Next(random, 2, maxSize);
+                var a = new BitArray(sz);
+                var b = new FixedBitSet(sz);
+
+                // test the various ways of setting bits
+                if (sz > 0)
+                {
+                    var nOper = random.Next(sz);
+                    for (var j = 0; j < nOper; j++)
+                    {
+                        int idx;
+
+                        idx = random.Next(sz);
+                        a.Set(idx);
+                        b.Set(idx);
+
+                        idx = random.Next(sz);
+                        a.Clear(idx);
+                        b.Clear(idx);
+
+                        idx = random.Next(sz);
+                        a.Flip(idx);
+                        b.Flip(idx, idx + 1);
+
+                        idx = random.Next(sz);
+                        a.Flip(idx);
+                        b.Flip(idx, idx + 1);
+                        bool val2 = b[idx];
+                        bool val = b.GetAndSet(idx);
+                        Assert.IsTrue(val2 == val);
+                        Assert.IsTrue(b[idx]);
+
+                        if (!val) b.Clear(idx);
+                        Assert.IsTrue(b[idx] == val);
+                    }
+                }
+
+                // test that the various ways of accessing the bits are equivalent
+                DoGet(a, b);
+
+                // test ranges, including possible extension
+                int fromIndex, toIndex;
+                fromIndex = random.Next(sz / 2);
+                toIndex = fromIndex + random.Next(sz - fromIndex);
+                var aa = (BitArray)a.Clone(); aa.Flip(fromIndex, toIndex);
+                var bb = b.Clone(); bb.Flip(fromIndex, toIndex);
+
+                DoIterate(aa, bb, mode);   // a problem here is from Flip or DoIterate
+
+                fromIndex = random.Next(sz / 2);
+                toIndex = fromIndex + random.Next(sz - fromIndex);
+                aa = (BitArray)a.Clone(); aa.Clear(fromIndex, toIndex);
+                bb = b.Clone(); bb.Clear(fromIndex, toIndex);
+
+                doNextSetBit(aa, bb); // a problem here is from Clear() or nextSetBit
+
+                doPrevSetBit(aa, bb);
+
+                fromIndex = random.Next(sz / 2);
+                toIndex = fromIndex + random.Next(sz - fromIndex);
+                aa = (BitArray)a.Clone(); aa.Set(fromIndex, toIndex);
+                bb = b.Clone(); bb.Set(fromIndex, toIndex);
+
+                doNextSetBit(aa, bb); // a problem here is from set() or nextSetBit
+
+                doPrevSetBit(aa, bb);
+
+                if (b0 != null && b0.Length <= b.Length)
+                {
+                    Assert.Equals(a.Cardinality(), b.Cardinality());
+
+                    var a_and = (BitArray)a.Clone(); a_and.And(a0);
+                    var a_or = (BitArray)a.Clone(); a_or.Or(a0);
+                    var a_andn = (BitArray)a.Clone(); a_andn.AndNot(a0);
+
+                    var b_and = b.Clone(); Assert.Equals(b, b_and); b_and.And(b0);
+                    var b_or = b.Clone(); b_or.Or(b0);
+                    var b_andn = b.Clone(); b_andn.AndNot(b0);
+
+                    Assert.Equals(a0.Cardinality(), b0.Cardinality());
+                    Assert.Equals(a_or.Cardinality(), b_or.Cardinality());
+
+                    DoIterate(a_and, b_and, mode);
+                    DoIterate(a_or, b_or, mode);
+                    DoIterate(a_andn, b_andn, mode);
+
+                    Assert.Equals(a_and.Cardinality(), b_and.Cardinality());
+                    Assert.Equals(a_or.Cardinality(), b_or.Cardinality());
+                    Assert.Equals(a_andn.Cardinality(), b_andn.Cardinality());
+                }
+
+                a0 = a;
+                b0 = b;
+            }
+        }
+
+        // large enough to flush obvious bugs, small enough to run in <.5 sec as part of a
+        // larger testsuite.
+        [Test]
+        public void TestSmall()
+        {
+            DoRandomSets(AtLeast(1200), AtLeast(1000), 1);
+            DoRandomSets(AtLeast(1200), AtLeast(1000), 2);
+        }
+
+        // uncomment to run a bigger test (~2 minutes).
+        /*
+        [Test]
+        public void TestBig() {
+          DoRandomSets(2000,200000, 1);
+          DoRandomSets(2000,200000, 2);
+        }
+        */
+
+        [Test]
+        public void TestEquals()
+        {
+            var random = new Random();
+
+            // This test can't handle numBits==0:
+            var numBits = random.Next(2000) + 1;
+            var b1 = new FixedBitSet(numBits);
+            var b2 = new FixedBitSet(numBits);
+            Assert.IsTrue(b1.Equals(b2));
+            Assert.IsTrue(b2.Equals(b1));
+            for (var iter = 0; iter < 10 * RANDOM_MULTIPLIER; iter++)
+            {
+                var idx = random.Next(numBits);
+                if (!b1[idx])
+                {
+                    b1.Set(idx);
+                    Assert.IsFalse(b1.Equals(b2));
+                    Assert.IsFalse(b2.Equals(b1));
+                    b2.Set(idx);
+                    Assert.IsTrue(b1.Equals(b2));
+                    Assert.IsTrue(b2.Equals(b1));
+                }
+            }
+
+            // try different type of object
+            Assert.IsFalse(b1.Equals(new Object()));
+        }
+
+        [Test]
+        public void TestHashCodeEquals()
+        {
+            var random = new Random();
+
+            // This test can't handle numBits==0:
+            var numBits = random.Next(2000) + 1;
+            var b1 = new FixedBitSet(numBits);
+            var b2 = new FixedBitSet(numBits);
+            Assert.IsTrue(b1.Equals(b2));
+            Assert.IsTrue(b2.Equals(b1));
+            for (var iter = 0; iter < 10 * RANDOM_MULTIPLIER; iter++)
+            {
+                int idx = random.Next(numBits);
+                if (!b1[idx])
+                {
+                    b1.Set(idx);
+                    Assert.IsFalse(b1.Equals(b2));
+                    Assert.IsFalse(b1.GetHashCode() == b2.GetHashCode());
+                    b2.Set(idx);
+                    Assert.Equals(b1, b2);
+                    Assert.Equals(b1.GetHashCode(), b2.GetHashCode());
+                }
+            }
+        }
+
+        [Test]
+        public void TestSmallBitSets()
+        {
+            // Make sure size 0-10 bit sets are OK:
+            for (var numBits = 0; numBits < 10; numBits++)
+            {
+                var b1 = new FixedBitSet(numBits);
+                var b2 = new FixedBitSet(numBits);
+                Assert.IsTrue(b1.Equals(b2));
+                Assert.Equals(b1.GetHashCode(), b2.GetHashCode());
+                Assert.Equals(0, b1.Cardinality());
+                if (numBits > 0)
+                {
+                    b1.Set(0, numBits);
+                    Assert.Equals(numBits, b1.Cardinality());
+                    b1.Flip(0, numBits);
+                    Assert.Equals(0, b1.Cardinality());
+                }
+            }
+        }
+
+        private FixedBitSet MakeFixedBitSet(int[] a, int numBits)
+        {
+            var random = new Random();
+            
+            FixedBitSet bs;
+            if (random.NextBool())
+            {
+                var bits2words = FixedBitSet.Bits2Words(numBits);
+                var words = new long[bits2words + random.Next(100)];
+                for (var i = bits2words; i < words.Length; i++)
+                {
+                    words[i] = random.NextLong();
+                }
+                bs = new FixedBitSet(words, numBits);
+
+            }
+            else
+            {
+                bs = new FixedBitSet(numBits);
+            }
+            foreach (var e in a)
+            {
+                bs.Set(e);
+            }
+            return bs;
+        }
+
+        private BitArray MakeBitSet(int[] a)
+        {
+            var bs = new BitArray();
+            foreach (var e in a)
+            {
+                bs.Set(e);
+            }
+            return bs;
+        }
+
+        private void CheckPrevSetBitArray(int[] a, int numBits)
+        {
+            var obs = MakeFixedBitSet(a, numBits);
+            var bs = MakeBitSet(a);
+            doPrevSetBit(bs, obs);
+        }
+
+        [Test]
+        public void TestPrevSetBit()
+        {
+            CheckPrevSetBitArray(new int[] { }, 0);
+            CheckPrevSetBitArray(new int[] { 0 }, 1);
+            CheckPrevSetBitArray(new int[] { 0, 2 }, 3);
+        }
+
+        private void CheckNextSetBitArray(int[] a, int numBits)
+        {
+            var obs = MakeFixedBitSet(a, numBits);
+            var bs = MakeBitSet(a);
+            doNextSetBit(bs, obs);
+        }
+
+        [Test]
+        public void TestNextBitSet()
+        {
+            var random = new Random();
+
+            var setBits = new int[0 + random.Next(1000)];
+            for (var i = 0; i < setBits.Length; i++)
+            {
+                setBits[i] = random.Next(setBits.Length);
+            }
+            CheckNextSetBitArray(setBits, setBits.Length + random.Next(10));
+
+            CheckNextSetBitArray(new int[0], setBits.Length + random.Next(10));
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/e02cc69c/test/core/Util/TestIdentityHashSet.cs
----------------------------------------------------------------------
diff --git a/test/core/Util/TestIdentityHashSet.cs b/test/core/Util/TestIdentityHashSet.cs
new file mode 100644
index 0000000..0c7e1e7
--- /dev/null
+++ b/test/core/Util/TestIdentityHashSet.cs
@@ -0,0 +1,45 @@
+using System;
+using System.Collections.Generic;
+using Lucene.Net.Support;
+using Lucene.Net.Util;
+using NUnit.Framework;
+
+namespace Lucene.Net.Test.Util
+{
+    [TestFixture]
+    public class TestIdentityHashSet : LuceneTestCase
+    {
+        public void testCheck()
+        {
+            var rnd = new Random();
+            ISet<object> jdk = Collections.NewSetFromMap(
+                new IdentityHashMap<object, bool>());
+            RamUsageEstimator.IdentityHashSet<object> us = new RamUsageEstimator.IdentityHashSet<object>();
+
+            var max = 100000;
+            var threshold = 256;
+            for (var i = 0; i < max; i++)
+            {
+                // some of these will be interned and some will not so there will be collisions.
+                var v = rnd.Next(threshold);
+
+                bool e1 = jdk.Contains(v);
+                bool e2 = us.Contains(v);
+                Assert.Equals(e1, e2);
+
+                e1 = jdk.Add(v);
+                e2 = us.Add(v);
+                Assert.Equals(e1, e2);
+            }
+
+            ISet<object> collected = Collections.NewSetFromMap(
+                new IdentityHashMap<object, bool>());
+            foreach (var o in us)
+            {
+                collected.Add(o);
+            }
+
+            Assert.Equals(collected, jdk);
+        }
+    }
+}


Mime
View raw message