lucenenet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mhern...@apache.org
Subject [1/5] git commit: adding all the methods required the BytesRef class and tests. Adding ICharSequence even though it is possible to use IEnumerable<char> instead.
Date Thu, 14 Aug 2014 01:22:23 GMT
Repository: lucenenet
Updated Branches:
  refs/heads/pcl 8e514a42b -> 7cfd98d54


adding all the methods required the BytesRef class and tests. Adding ICharSequence even though it is possible to use IEnumerable<char> instead.


Project: http://git-wip-us.apache.org/repos/asf/lucenenet/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucenenet/commit/626809a0
Tree: http://git-wip-us.apache.org/repos/asf/lucenenet/tree/626809a0
Diff: http://git-wip-us.apache.org/repos/asf/lucenenet/diff/626809a0

Branch: refs/heads/pcl
Commit: 626809a0cb9cabf54527773693a8ef222bbabc1b
Parents: 8e514a4
Author: Michael Herndon <mherndon@michaelherndon.com>
Authored: Tue Aug 12 17:14:25 2014 -0400
Committer: Michael Herndon <mherndon@michaelherndon.com>
Committed: Tue Aug 12 17:14:25 2014 -0400

----------------------------------------------------------------------
 src/Lucene.Net.Core/Lucene.Net.Core.csproj      |   5 +
 src/Lucene.Net.Core/Support/ICharSequence.cs    |  17 +
 .../Support/NumberExtensionMethods.cs           |  29 ++
 src/Lucene.Net.Core/Util/ArrayUtil.cs           | 137 +++++-
 src/Lucene.Net.Core/Util/BroadWord.cs           |  32 +-
 src/Lucene.Net.Core/Util/BytesRef.cs            | 481 +++++++++++++++++++
 src/Lucene.Net.Core/Util/CharsRef.cs            | 414 ++++++++++++++++
 src/Lucene.Net.Core/Util/RamUsageEstimator.cs   |  40 +-
 src/Lucene.Net.Core/Util/StringHelper.cs        |  87 ++++
 src/Lucene.Net.Core/Util/UnicodeUtil.cs         | 145 ++++++
 .../Lucene.Net.Core.Tests.csproj                |   2 +
 .../Util/BaseSorterTestCase.cs                  |   3 +-
 test/Lucene.Net.Core.Tests/Util/TestBits.cs     |   1 -
 test/Lucene.Net.Core.Tests/Util/TestBytesRef.cs |  49 ++
 test/Lucene.Net.Core.Tests/Util/TestCharsRef.cs |  25 +
 .../Lucene.Net.Core.Tests/Util/TestConstants.cs |   1 -
 .../Util/TestRamEstimatorUsage.cs               |   1 -
 test/Lucene.Net.Core.Tests/Util/TestVersion.cs  |   1 -
 .../Lucene.Net.TestFramework.csproj             |   2 +
 .../Random/RandomExtensions.cs                  |  17 +-
 .../Util/ExtensionMethods.cs                    |  40 ++
 .../Util/LuceneTestCase.cs                      |   5 +
 test/Lucene.Net.TestFramework/Util/TestUtil.cs  |  90 ++++
 23 files changed, 1581 insertions(+), 43 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucenenet/blob/626809a0/src/Lucene.Net.Core/Lucene.Net.Core.csproj
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Core/Lucene.Net.Core.csproj b/src/Lucene.Net.Core/Lucene.Net.Core.csproj
index aed403a..4e1050a 100644
--- a/src/Lucene.Net.Core/Lucene.Net.Core.csproj
+++ b/src/Lucene.Net.Core/Lucene.Net.Core.csproj
@@ -61,6 +61,7 @@
     <Compile Include="Support\DeepCloneNotSupportedException.cs" />
     <Compile Include="Support\EnumUtil.cs" />
     <Compile Include="Support\HashMap.cs" />
+    <Compile Include="Support\ICharSequence.cs" />
     <Compile Include="Support\ICloneable.cs" />
     <Compile Include="Support\NumberExtensionMethods.cs" />
     <Compile Include="Support\SerializableAttribute.cs" />
@@ -72,6 +73,8 @@
     <Compile Include="Util\Bits.cs" />
     <Compile Include="Util\BitUtil.cs" />
     <Compile Include="Util\BroadWord.cs" />
+    <Compile Include="Util\BytesRef.cs" />
+    <Compile Include="Util\CharsRef.cs" />
     <Compile Include="Util\Constants.cs" />
     <Compile Include="Util\IAccountable.cs" />
     <Compile Include="Util\IAttribute.cs" />
@@ -79,7 +82,9 @@
     <Compile Include="Util\InPlaceMergeSorter.cs" />
     <Compile Include="Util\RamUsageEstimator.cs" />
     <Compile Include="Util\Sorter.cs" />
+    <Compile Include="Util\StringHelper.cs" />
     <Compile Include="Util\SystemProps.cs" />
+    <Compile Include="Util\UnicodeUtil.cs" />
     <Compile Include="Util\Version.cs" />
   </ItemGroup>
   <ItemGroup>

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/626809a0/src/Lucene.Net.Core/Support/ICharSequence.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Core/Support/ICharSequence.cs b/src/Lucene.Net.Core/Support/ICharSequence.cs
new file mode 100644
index 0000000..8dbad2e
--- /dev/null
+++ b/src/Lucene.Net.Core/Support/ICharSequence.cs
@@ -0,0 +1,17 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Lucene.Net.Support
+{
+    public interface ICharSequence
+    {
+        int Length { get; }
+
+        char CharAt(int index);
+
+        ICharSequence SubSequence(int start, int end);
+    }
+}

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/626809a0/src/Lucene.Net.Core/Support/NumberExtensionMethods.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Core/Support/NumberExtensionMethods.cs b/src/Lucene.Net.Core/Support/NumberExtensionMethods.cs
index c115bdc..53971ed 100644
--- a/src/Lucene.Net.Core/Support/NumberExtensionMethods.cs
+++ b/src/Lucene.Net.Core/Support/NumberExtensionMethods.cs
@@ -112,5 +112,34 @@ namespace Lucene.Net.Support
         {
             return DeBruijn64.Position(value);
         }
+
+
+        internal static uint RotateLeft(this uint value, int distance)
+        {
+            var v = value;
+            return (v << distance) | (v >> (32 - distance));
+        }
+
+        internal static uint RotateRight(this uint value, int distance)
+        {
+            var v = value;
+            return (v >> distance) | (v << (32 - distance));
+        }
+
+        /// <summary>
+        /// 
+        /// </summary>
+        /// <param name="value"></param>
+        /// <param name="distance"></param>
+        /// <returns></returns>
+        public static int RotateLeft(this int value, int distance)
+        {
+            return (int)RotateLeft((uint)value, distance);
+        }
+
+        public static int RotateRight(this int value, int distance)
+        {
+            return (int)RotateRight((uint)value, distance);
+        }
     }
 }

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/626809a0/src/Lucene.Net.Core/Util/ArrayUtil.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Core/Util/ArrayUtil.cs b/src/Lucene.Net.Core/Util/ArrayUtil.cs
index 9cde720..091ad0a 100644
--- a/src/Lucene.Net.Core/Util/ArrayUtil.cs
+++ b/src/Lucene.Net.Core/Util/ArrayUtil.cs
@@ -1,9 +1,12 @@
 
 
-namespace Lucene.Net
+namespace Lucene.Net.Util
 {
     using System;
     using System.Collections.Generic;
+    using System.Linq;
+    using System.Diagnostics;
+    using System.Reflection;
 
     /// <summary>
     /// Utility methods for manipulating arrays.
@@ -13,6 +16,138 @@ namespace Lucene.Net
 
 
         /// <summary>
+        /// Returns an array that has been resized to a length greater than
+        /// the original array.
+        /// </summary>
+        /// <typeparam name="T">The element type for the array.</typeparam>
+        /// <param name="array">The array to base the resize on.</param>
+        /// <param name="minSize">The minimum size to grow the array.</param>
+        /// <returns>The resized array.</returns>
+        /// <exception cref="ArgumentException">Throws when <paramref name="minTargetSize"/> is less than zero.</exception>
+        public static T[] Grow<T>(this T[] array, int minSize = 1)
+        {
+            Debug.Assert(typeof(T).GetTypeInfo().IsPrimitive, "Type T must be primitive");
+            Debug.Assert(minSize >= 0, "targetSize must be positive");
+
+            if (array.Length < minSize)
+            {
+                int capacity = Oversize(minSize, RamUsageEstimator.PrimitiveSizes[typeof(T)]);
+                var oversizedArray = new T[capacity];
+                Array.Copy(array, 0, oversizedArray, 0, array.Length);
+
+                return oversizedArray;
+            }
+            else
+            {
+                return array;
+            }
+        }
+
+
+        /// <summary>
+        /// Returns a hashcode for an array.
+        /// </summary>
+        /// <typeparam name="T">The element type for the array.</typeparam>
+        /// <param name="array">The target array to create a hashcode.</param>
+        /// <param name="start">The starting position for a loop.</param>
+        /// <param name="end">The number of iterations for the loop to perform.</param>
+        /// <returns>The hashcode.</returns>
+        public static int CreateHashCode<T>(this T[] array, int start, int end) where T : struct
+        {
+            var code = 0;
+            var elements = array.Cast<int>().ToArray();
+            for (var i = end - 1; i >= start; i--)
+            {
+                code = code * 31 + elements[i];
+            }
+
+            return code;
+        }
+
+
+        /// <summary>
+        /// Returns a new capacity number to resize an array.
+        /// </summary>
+        /// <param name="minTargetSize">The minium size for the oversize.</param>
+        /// <param name="bytesPerElement">The number of bytes an element of the array will allocate in memory.</param>
+        /// <returns>The new capacity size.</returns>
+        /// <exception cref="ArgumentException">Throws when <paramref name="minTargetSize"/> is less than zero.</exception>
+        public static int Oversize(int minTargetSize, int bytesPerElement)
+        {
+            // catch usage that accidentally overflows int
+            Check.Condition(minTargetSize < 0, "minTargetSize", "invalid array size {0}", minTargetSize);
+
+            if (minTargetSize == 0)
+            {
+                // wait until at least one element is requested
+                return 0;
+            }
+
+            // asymptotic exponential growth by 1/8th, favors
+            // spending a bit more CPU to not tie up too much wasted
+            // RAM:
+            int extra = minTargetSize >> 3;
+
+            if (extra < 3)
+            {
+                // for very small arrays, where constant overhead of
+                // realloc is presumably relatively high, we grow
+                // faster
+                extra = 3;
+            }
+
+            int newSize = minTargetSize + extra;
+
+            // add 7 to allow for worst case byte alignment addition below:
+            if (newSize + 7 < 0)
+            {
+                // int overflowed -- return max allowed array size
+                return int.MaxValue;
+            }
+
+            if (Constants.KRE_IS_64BIT)
+            {
+                // round up to 8 byte alignment in 64bit env
+                switch (bytesPerElement)
+                {
+                    case 4:
+                        // round up to multiple of 2
+                        return (newSize + 1) & 0x7ffffffe;
+                    case 2:
+                        // round up to multiple of 4
+                        return (newSize + 3) & 0x7ffffffc;
+                    case 1:
+                        // round up to multiple of 8
+                        return (newSize + 7) & 0x7ffffff8;
+                    case 8:
+                    // no rounding
+                    default:
+                        // odd (invalid?) size
+                        return newSize;
+                }
+            }
+            else
+            {
+                // round up to 4 byte alignment in 64bit env
+                switch (bytesPerElement)
+                {
+                    case 2:
+                        // round up to multiple of 2
+                        return (newSize + 1) & 0x7ffffffe;
+                    case 1:
+                        // round up to multiple of 4
+                        return (newSize + 3) & 0x7ffffffc;
+                    case 4:
+                    case 8:
+                    // no rounding
+                    default:
+                        // odd (invalid?) size
+                        return newSize;
+                }
+            }
+        }
+
+        /// <summary>
         /// Parses the string argument as if it was an int value and returns the result.
         /// </summary>
         /// <param name="chars">A string representation of an int quantity. </param>

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/626809a0/src/Lucene.Net.Core/Util/BroadWord.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Core/Util/BroadWord.cs b/src/Lucene.Net.Core/Util/BroadWord.cs
index de2553b..074c82a 100644
--- a/src/Lucene.Net.Core/Util/BroadWord.cs
+++ b/src/Lucene.Net.Core/Util/BroadWord.cs
@@ -24,7 +24,7 @@ namespace Lucene.Net.Util
     using Lucene.Net.Support;
 
     /// <summary>
-    ///
+    /// An implementation of Broadword for rank/select queries.
     /// </summary>
     /// <remarks>
     ///     <para>
@@ -48,34 +48,35 @@ namespace Lucene.Net.Util
         // Thus, there is not a valid reason to make them public.  
 
         /// <summary>
-        ///  L8  denotes the constant of 8-byte-counts or 8k.
-        ///  _L denotes that the number is an long format. 
+        /// L8  denotes the constant of 8-byte-counts or 8k.
+        /// _L denotes that the number is an long format. 
         /// </summary>
         private const ulong L8_L = 0x0101010101010101L;
 
         /// <summary>
-        ///  L9  denotes the constant of 8-byte-counts or 9k.
-        ///  _L denotes that the number is an long format. 
+        /// L9  denotes the constant of 8-byte-counts or 9k.
+        /// _L denotes that the number is an long format. 
         /// </summary>
         private const ulong L9_L = 0x8040201008040201L;
 
         /// <summary>
-        ///  L16  denotes the constant of 16-byte-counts or 16k.
-        ///  _L denotes that the number is an long format. 
+        /// L16  denotes the constant of 16-byte-counts or 16k.
+        /// _L denotes that the number is an long format. 
         /// </summary>
         private const ulong L16_L = 0x0001000100010001L;
 
         /// <summary>
         /// H8 = L8 << (8-1) .
-        ///  These contain the high bit of each group of k bits.
-        ///  The suffix _L indicates the long implementation.
+        /// These contain the high bit of each group of k bits.
+        /// The suffix _L indicates the long implementation.
         /// </summary>
         private static readonly ulong H8_L = L8_L << 7;
 
+        /* unused
         /// H16 = L16 << (16-1) .
         ///  These contain the high bit of each group of k bits.
         ///  The suffix _L indicates the long implementation.
-        private static readonly ulong H16_L = L16_L << 15;
+        private static readonly ulong H16_L = L16_L << 15; */
 
         /// <summary>
         /// Bit count of a long.
@@ -96,8 +97,9 @@ namespace Lucene.Net.Util
         }
 
        
-         /// <summary>
-        /// Select a 1-bit from a long. </summary>
+        /// <summary>
+        /// Select a 1-bit from a long. 
+        /// </summary>
         /// <returns> The index of the right most 1 bit in x, or if no such bit exists, 72. </returns>
         public static int Select(long value, int rank)
         {
@@ -136,7 +138,8 @@ namespace Lucene.Net.Util
      
         /// <summary>
         /// A signed bytewise smaller &lt;<sub><small>8</small></sub> operator, for operands 0L<= x, y <=0x7L.
-        /// this uses the following numbers of basic long operations: 1 or, 2 and, 2 xor, 1 minus, 1 not. </summary>
+        /// this uses the following numbers of basic long operations: 1 or, 2 and, 2 xor, 1 minus, 1 not. 
+        /// </summary>
         /// <returns> A long with bits set in the <seealso cref="#H8_L"/> positions corresponding to each input signed byte pair that compares smaller. </returns>
         private static ulong SmallerUpTo7_8(ulong x, ulong y)
         {
@@ -151,7 +154,8 @@ namespace Lucene.Net.Util
 
         /// <summary>
         /// Naive implementation of <seealso cref="#select(long,int)"/>, using <seealso cref="Long#numberOfTrailingZeros"/> repetitively.
-        /// Works relatively fast for low ranks. </summary>
+        /// Works relatively fast for low ranks. 
+        /// </summary>
         /// <returns> The index of the r-th 1 bit in x, or if no such bit exists, 72. </returns>
         public static int SelectNaive(long x, int r)
         {

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/626809a0/src/Lucene.Net.Core/Util/BytesRef.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Core/Util/BytesRef.cs b/src/Lucene.Net.Core/Util/BytesRef.cs
new file mode 100644
index 0000000..b400896
--- /dev/null
+++ b/src/Lucene.Net.Core/Util/BytesRef.cs
@@ -0,0 +1,481 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+namespace Lucene.Net.Util
+{
+    using Lucene.Net.Support;
+    using System;
+    using System.Collections.Generic;
+    using System.Diagnostics;
+    using System.Text;
+
+    /// <summary>
+    /// Represents byte[], as a slice (offset + length) into an
+    ///  existing byte[].  The <seealso cref="#bytes"/> member should never be null;
+    ///  use <seealso cref="#EMPTY_BYTES"/> if necessary.
+    ///
+    /// <p><b>Important note:</b> Unless otherwise noted, Lucene uses this class to
+    /// represent terms that are encoded as <b>UTF8</b> bytes in the index. To
+    /// convert them to a Java <seealso cref="String"/> (which is UTF16), use <seealso cref="#Utf8ToString"/>.
+    /// Using code like {@code new String(bytes, offset, length)} to do this
+    /// is <b>wrong</b>, as it does not respect the correct character set
+    /// and may return wrong results (depending on the platform's defaults)!
+    /// </summary>
+    public sealed class BytesRef : 
+        System.IComparable, 
+        Lucene.Net.Support.ICloneable,
+        IEnumerable<Byte>
+    {
+        /// <summary>
+        /// An empty byte array for convenience </summary>
+        public static readonly byte[] EMPTY_BYTES = new byte[0];
+
+        /// <summary>
+        /// The contents of the BytesRef. Should never be {@code null}. </summary>
+        public byte[] Bytes { get; set; }
+
+        /// <summary>
+        /// Offset of first valid byte. </summary>
+        public int Offset { get; set; }
+
+        /// <summary>
+        /// Length of used bytes. </summary>
+        public int Length { get; set; }
+
+        /// <summary>
+        /// Create a BytesRef with <seealso cref="#EMPTY_BYTES"/> </summary>
+        public BytesRef()
+            : this(EMPTY_BYTES)
+        {
+        }
+
+        /// <summary>
+        /// this instance will directly reference bytes w/o making a copy.
+        /// bytes should not be null.
+        /// </summary>
+        public BytesRef(byte[] bytes, int offset, int length)
+        {
+            this.Bytes = bytes;
+            this.Offset = offset;
+            this.Length = length;
+            Debug.Assert(Valid);
+        }
+
+        /// <summary>
+        /// this instance will directly reference bytes w/o making a copy.
+        /// bytes should not be null
+        /// </summary>
+        public BytesRef(byte[] bytes)
+            : this(bytes, 0, bytes.Length)
+        {
+        }
+
+        /// <summary>
+        /// Create a BytesRef pointing to a new array of size <code>capacity</code>.
+        /// Offset and length will both be zero.
+        /// </summary>
+        public BytesRef(int capacity)
+        {
+            this.Bytes = new byte[capacity];
+        }
+
+        /// <summary>
+        /// Initialize the byte[] from the UTF8 bytes
+        /// for the provided String.
+        /// </summary>
+        /// <param name="text"> this must be well-formed
+        /// unicode text, with no unpaired surrogates. </param>
+        public BytesRef(CharsRef text)
+            : this()
+        {
+            CopyChars(text);
+        }
+
+        /// <summary>
+        /// Initialize the byte[] from the UTF8 bytes
+        /// for the provided String.
+        /// </summary>
+        /// <param name="text"> this must be well-formed
+        /// unicode text, with no unpaired surrogates. </param>
+        public BytesRef(string text)
+            : this()
+        {
+            CopyChars(text);
+        }
+
+        /// <summary>
+        /// Copies the UTF8 bytes for this string.
+        /// </summary>
+        /// <param name="text"> Must be well-formed unicode text, with no
+        /// unpaired surrogates or invalid UTF16 code units. </param>
+        public void CopyChars(CharsRef text)
+        {
+            Debug.Assert(this.Offset == 0); // TODO broken if offset != 0
+            UnicodeUtil.UTF16toUTF8(text, 0, text.Length, this);
+        }
+
+        /// <summary>
+        /// Copies the UTF8 bytes for this string.
+        /// </summary>
+        /// <param name="text"> Must be well-formed unicode text, with no
+        /// unpaired surrogates or invalid UTF16 code units. </param>
+        public void CopyChars(string text)
+        {
+            Debug.Assert(this.Offset == 0); // TODO broken if offset != 0
+            UnicodeUtil.UTF16toUTF8(text.ToCharArray(0, text.Length), 0, text.Length, this);
+        }
+
+        /// <summary>
+        /// Expert: compares the bytes against another BytesRef,
+        /// returning true if the bytes are equal.
+        /// </summary>
+        /// <param name="other"> Another BytesRef, should not be null.
+        /// @lucene.internal </param>
+        public bool BytesEquals(BytesRef other)
+        {
+            Debug.Assert(other != null);
+            if (Length == other.Length)
+            {
+                int otherUpto = other.Offset;
+                byte[] otherBytes = other.Bytes;
+                int end = this.Offset + this.Length;
+                for (int upto = Offset; upto < end; upto++, otherUpto++)
+                {
+                    if (this.Bytes[upto] != otherBytes[otherUpto])
+                    {
+                        return false;
+                    }
+                }
+                return true;
+            }
+            else
+            {
+                return false;
+            }
+        }
+
+        /// <summary>
+        /// Returns a shallow clone of this instance (the underlying bytes are
+        /// <b>not</b> copied and will be shared by both the returned object and this
+        /// object.
+        /// </summary>
+        /// <seealso cref= #deepCopyOf </seealso>
+        public object Clone(bool deepClone = false)
+        {
+            if (deepClone)
+                throw new DeepCloneNotSupportedException(typeof(CharsRef));
+
+            return new BytesRef(this.Bytes, this.Offset, this.Length);
+        }
+
+        /// <summary>
+        /// Calculates the hash code as required by TermsHash during indexing.
+        ///  <p> this is currently implemented as MurmurHash3 (32
+        ///  bit), using the seed from {@link
+        ///  StringHelper#GOOD_FAST_HASH_SEED}, but is subject to
+        ///  change from release to release.
+        /// </summary>
+        public override int GetHashCode()
+        {
+            return StringHelper.MurmurHash3_x86_32(this.Bytes, this.Offset, this.Length, StringHelper.GOOD_FAST_HASH_SEED);
+        }
+
+        public override bool Equals(object other)
+        {
+            if (other == null)
+            {
+                return false;
+            }
+            if (other is BytesRef)
+            {
+                return this.BytesEquals((BytesRef)other);
+            }
+            return false;
+        }
+
+        /// <summary>
+        /// Interprets stored bytes as UTF8 bytes, returning the
+        ///  resulting string
+        /// </summary>
+        public string Utf8ToString()
+        {
+            CharsRef @ref = new CharsRef(Length);
+            UnicodeUtil.UTF8toUTF16(this.Bytes, this.Offset, this.Length, @ref);
+            return @ref.ToString();
+        }
+
+        /// <summary>
+        /// Returns hex encoded bytes, eg [0x6c 0x75 0x63 0x65 0x6e 0x65] </summary>
+        public override string ToString()
+        {
+            StringBuilder sb = new StringBuilder();
+            sb.Append('[');
+            int end = this.Offset + Length;
+            for (int i = this.Offset; i < end; i++)
+            {
+                if (i > this.Offset)
+                {
+                    sb.Append(' ');
+                }
+                sb.Append((this.Bytes[i] & 0xff).ToString("x"));
+            }
+            sb.Append(']');
+            return sb.ToString();
+        }
+
+        /// <summary>
+        /// Copies the bytes from the given <seealso cref="BytesRef"/>
+        /// <p>
+        /// NOTE: if this would exceed the array size, this method creates a
+        /// new reference array.
+        /// </summary>
+        public void CopyBytes(BytesRef other)
+        {
+            if (this.Bytes.Length - this.Offset < other.Length)
+            {
+                this.Bytes = new byte[other.Length];
+                this.Offset = 0;
+            }
+            Array.Copy(other.Bytes, other.Offset, Bytes, Offset, other.Length);
+            this.Length = other.Length;
+        }
+
+        /// <summary>
+        /// Appends the bytes from the given <seealso cref="BytesRef"/>
+        /// <p>
+        /// NOTE: if this would exceed the array size, this method creates a
+        /// new reference array.
+        /// </summary>
+        public void Append(BytesRef other)
+        {
+            int newLen = Length + other.Length;
+            if (this.Bytes.Length - this.Offset < newLen)
+            {
+                byte[] newBytes = new byte[newLen];
+                Array.Copy(this.Bytes, this.Offset, newBytes, 0, Length);
+                Offset = 0;
+                Bytes = newBytes;
+            }
+            Array.Copy(other.Bytes, other.Offset, Bytes, Length + Offset, other.Length);
+            this.Length = newLen;
+        }
+
+        /// <summary>
+        /// Used to grow the reference array.
+        ///
+        /// In general this should not be used as it does not take the offset into account.
+        /// @lucene.internal
+        /// </summary>
+        public void Grow(int newLength)
+        {
+            Debug.Assert(this.Offset == 0); // NOTE: senseless if offset != 0
+            this.Bytes = ArrayUtil.Grow(this.Bytes, newLength);
+        }
+
+        /// <summary>
+        /// Unsigned byte order comparison </summary>
+        public int CompareTo(object other)
+        {
+            BytesRef br = other as BytesRef;
+            Debug.Assert(br != null);
+            return Utf8SortedAsUnicodeSortOrder.Compare(this, br);
+        }
+
+        private static readonly IComparer<BytesRef> Utf8SortedAsUnicodeSortOrder = new UTF8SortedAsUnicodeComparator();
+
+        public static IComparer<BytesRef> UTF8SortedAsUnicodeComparer
+        {
+            get
+            {
+                return Utf8SortedAsUnicodeSortOrder;
+            }
+        }
+
+        private class UTF8SortedAsUnicodeComparator : IComparer<BytesRef>
+        {
+            // Only singleton
+            internal UTF8SortedAsUnicodeComparator()
+            {
+            }
+
+            public virtual int Compare(BytesRef a, BytesRef b)
+            {
+                byte[] aBytes = a.Bytes;
+                int aUpto = a.Offset;
+                byte[] bBytes = b.Bytes;
+                int bUpto = b.Offset;
+
+                int aStop = aUpto + Math.Min(a.Length, b.Length);
+                while (aUpto < aStop)
+                {
+                    int aByte = aBytes[aUpto++] & 0xff;
+                    int bByte = bBytes[bUpto++] & 0xff;
+
+                    int diff = aByte - bByte;
+                    if (diff != 0)
+                    {
+                        return diff;
+                    }
+                }
+
+                // One is a prefix of the other, or, they are equal:
+                return a.Length - b.Length;
+            }
+        }
+
+        /// @deprecated this comparator is only a transition mechanism
+#pragma warning disable 0612, 0618
+        private static readonly IComparer<BytesRef> Utf8SortedAsUTF16SortOrder = new UTF8SortedAsUTF16Comparator();
+#pragma warning restore 0612, 0618
+
+        /// @deprecated this comparator is only a transition mechanism
+        [Obsolete("this comparator is only a transition mechanism")]
+        public static IComparer<BytesRef> UTF8SortedAsUTF16Comparer
+        {
+            get
+            {
+                return Utf8SortedAsUTF16SortOrder;
+            }
+        }
+
+        /// @deprecated this comparator is only a transition mechanism
+        [Obsolete("this comparator is only a transition mechanism")]
+        private class UTF8SortedAsUTF16Comparator : IComparer<BytesRef>
+        {
+            // Only singleton
+            internal UTF8SortedAsUTF16Comparator()
+            {
+            }
+
+            public virtual int Compare(BytesRef a, BytesRef b)
+            {
+                byte[] aBytes = a.Bytes;
+                int aUpto = a.Offset;
+                byte[] bBytes = b.Bytes;
+                int bUpto = b.Offset;
+
+                int aStop;
+                if (a.Length < b.Length)
+                {
+                    aStop = aUpto + a.Length;
+                }
+                else
+                {
+                    aStop = aUpto + b.Length;
+                }
+
+                while (aUpto < aStop)
+                {
+                    int aByte = aBytes[aUpto++] & 0xff;
+                    int bByte = bBytes[bUpto++] & 0xff;
+
+                    if (aByte != bByte)
+                    {
+                        // See http://icu-project.org/docs/papers/utf16_code_point_order.html#utf-8-in-utf-16-order
+
+                        // We know the terms are not equal, but, we may
+                        // have to carefully fixup the bytes at the
+                        // difference to match UTF16's sort order:
+
+                        // NOTE: instead of moving supplementary code points (0xee and 0xef) to the unused 0xfe and 0xff,
+                        // we move them to the unused 0xfc and 0xfd [reserved for future 6-byte character sequences]
+                        // this reserves 0xff for preflex's term reordering (surrogate dance), and if unicode grows such
+                        // that 6-byte sequences are needed we have much bigger problems anyway.
+                        if (aByte >= 0xee && bByte >= 0xee)
+                        {
+                            if ((aByte & 0xfe) == 0xee)
+                            {
+                                aByte += 0xe;
+                            }
+                            if ((bByte & 0xfe) == 0xee)
+                            {
+                                bByte += 0xe;
+                            }
+                        }
+                        return aByte - bByte;
+                    }
+                }
+
+                // One is a prefix of the other, or, they are equal:
+                return a.Length - b.Length;
+            }
+        }
+
+        /// <summary>
+        /// Creates a new BytesRef that points to a copy of the bytes from
+        /// <code>other</code>
+        /// <p>
+        /// The returned BytesRef will have a length of other.length
+        /// and an offset of zero.
+        /// </summary>
+        public static BytesRef DeepCopyOf(BytesRef other)
+        {
+            BytesRef copy = new BytesRef();
+            copy.CopyBytes(other);
+            return copy;
+        }
+
+        /// <summary>
+        /// Performs internal consistency checks.
+        /// Always returns true (or throws InvalidOperationException)
+        /// </summary>
+        public bool Valid
+        {
+            get
+            {
+                if (Bytes == null)
+                {
+                    throw new Exception("bytes is null");
+                }
+                if (Length < 0)
+                {
+                    throw new Exception("length is negative: " + Length);
+                }
+                if (Length > Bytes.Length)
+                {
+                    throw new Exception("length is out of bounds: " + Length + ",bytes.length=" + Bytes.Length);
+                }
+                if (Offset < 0)
+                {
+                    throw new Exception("offset is negative: " + Offset);
+                }
+                if (Offset > Bytes.Length)
+                {
+                    throw new Exception("offset out of bounds: " + Offset + ",bytes.length=" + Bytes.Length);
+                }
+                if (Offset + Length < 0)
+                {
+                    throw new Exception("offset+length is negative: offset=" + Offset + ",length=" + Length);
+                }
+                if (Offset + Length > Bytes.Length)
+                {
+                    throw new Exception("offset+length out of bounds: offset=" + Offset + ",length=" + Length + ",bytes.length=" + Bytes.Length);
+                }
+                return true;
+            }
+        }
+
+        public IEnumerator<byte> GetEnumerator()
+        {
+            return ((IEnumerable<byte>)this.Bytes).GetEnumerator();
+        }
+
+        System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator()
+        {
+            return this.Bytes.GetEnumerator();
+        }
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/626809a0/src/Lucene.Net.Core/Util/CharsRef.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Core/Util/CharsRef.cs b/src/Lucene.Net.Core/Util/CharsRef.cs
new file mode 100644
index 0000000..53150e5
--- /dev/null
+++ b/src/Lucene.Net.Core/Util/CharsRef.cs
@@ -0,0 +1,414 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+namespace Lucene.Net.Util
+{
+    using Lucene.Net.Support;
+    using System;
+    using System.Collections.Generic;
+    using System.Diagnostics;
+    using System.Linq;
+
+    /// <summary>
+    /// Represents char[], as a slice (offset + length) into an existing char[].
+    /// The <seealso cref="#chars"/> member should never be null; use
+    /// <seealso cref="#EMPTY_CHARS"/> if necessary.
+    /// @lucene.internal
+    /// </summary>
+    public sealed class CharsRef : IComparable<CharsRef>, 
+        ICharSequence, 
+        Lucene.Net.Support.ICloneable, 
+        IEnumerable<char>
+    {
+       
+
+        /// <summary>
+        /// An empty character array for convenience </summary>
+        public static readonly char[] EMPTY_CHARS = new char[0];
+
+        /// <summary>
+        /// The contents of the CharsRef. Should never be <see cref="Null"/>. </summary>
+        public char[] Chars { get; set; }
+
+        /// <summary>
+        /// Offset of first valid character. </summary>
+        public int Offset {get; set;}
+
+        /// <summary>
+        /// Length of used characters. </summary>
+        public int Length { get; internal set; }
+
+     
+        /// <summary>
+        /// Creates a new <seealso cref="CharsRef"/> initialized an empty array zero-length
+        /// </summary>
+        public CharsRef()
+            : this(EMPTY_CHARS, 0, 0)
+        {
+        }
+
+        /// <summary>
+        /// Creates a new <seealso cref="CharsRef"/> initialized with an array of the given
+        /// capacity
+        /// </summary>
+        public CharsRef(int capacity)
+        {
+            this.Chars = new char[capacity];
+        }
+
+        /// <summary>
+        /// Creates a new <seealso cref="CharsRef"/> initialized with the given array, offset and
+        /// length
+        /// </summary>
+        public CharsRef(char[] chars, int offset, int length)
+        {
+            this.Chars = chars;
+            this.Offset = offset;
+            this.Length = length;
+            Debug.Assert(Valid);
+        }
+
+        /// <summary>
+        /// Creates a new <seealso cref="CharsRef"/> initialized with the given Strings character
+        /// array
+        /// </summary>
+        public CharsRef(string @string)
+        {
+            this.Chars = @string.ToCharArray();
+            this.Offset = 0;
+            this.Length = Chars.Length;
+        }
+
+        /// <summary>
+        /// Returns a shallow clone of this instance (the underlying characters are
+        /// <b>not</b> copied and will be shared by both the returned object and this
+        /// object.
+        /// </summary>
+        /// <seealso cref= #deepCopyOf </seealso>
+        public object Clone(bool deepClone = false)
+        {
+            if (deepClone)
+                throw new DeepCloneNotSupportedException(typeof(CharsRef));
+
+            return new CharsRef(Chars, Offset, Length);
+        }
+
+        public override int GetHashCode()
+        {
+            const int prime = 31;
+            int result = 0;
+            int end = Offset + Length;
+            for (int i = Offset; i < end; i++)
+            {
+                result = prime * result + Chars[i];
+            }
+            return result;
+        }
+
+        public override bool Equals(object other)
+        {
+            if (other == null)
+            {
+                return false;
+            }
+            if (other is CharsRef)
+            {
+                return this.CharsEquals((CharsRef)other);
+            }
+            return false;
+        }
+
+        public bool CharsEquals(CharsRef other)
+        {
+            if (Length == other.Length)
+            {
+                int otherUpto = other.Offset;
+                char[] otherChars = other.Chars;
+                int end = this.Offset + this.Length;
+                for (int upto = Offset; upto < end; upto++, otherUpto++)
+                {
+                    if (this.Chars[upto] != otherChars[otherUpto])
+                    {
+                        return false;
+                    }
+                }
+                return true;
+            }
+            else
+            {
+                return false;
+            }
+        }
+
+        /// <summary>
+        /// Signed int order comparison </summary>
+        public int CompareTo(CharsRef other)
+        {
+            if (this == other)
+            {
+                return 0;
+            }
+
+            char[] aChars = this.Chars;
+            int aUpto = this.Offset;
+            char[] bChars = other.Chars;
+            int bUpto = other.Offset;
+
+            int aStop = aUpto + Math.Min(this.Length, other.Length);
+
+            while (aUpto < aStop)
+            {
+                int aInt = aChars[aUpto++];
+                int bInt = bChars[bUpto++];
+                if (aInt > bInt)
+                {
+                    return 1;
+                }
+                else if (aInt < bInt)
+                {
+                    return -1;
+                }
+            }
+
+            // One is a prefix of the other, or, they are equal:
+            return this.Length - other.Length;
+        }
+
+        /// <summary>
+        /// Copies the given <seealso cref="CharsRef"/> referenced content into this instance.
+        /// </summary>
+        /// <param name="other">
+        ///          the <seealso cref="CharsRef"/> to copy </param>
+        public void CopyChars(CharsRef other)
+        {
+            CopyChars(other.Chars, other.Offset, other.Length);
+        }
+
+        /// <summary>
+        /// Used to grow the reference array.
+        ///
+        /// In general this should not be used as it does not take the offset into account.
+        /// @lucene.internal
+        /// </summary>
+        public void Grow(int newLength)
+        {
+            Debug.Assert(Offset == 0);
+            if (this.Chars.Length < newLength)
+            {
+                this.Chars = ArrayUtil.Grow(this.Chars, newLength);
+            }
+        }
+
+        /// <summary>
+        /// Copies the given array into this CharsRef.
+        /// </summary>
+        public void CopyChars(char[] otherChars, int otherOffset, int otherLength)
+        {
+            if (this.Chars.Length - this.Offset < otherLength)
+            {
+                this.Chars = new char[otherLength];
+                this.Offset = 0;
+            }
+            Array.Copy(otherChars, otherOffset, this.Chars, this.Offset, otherLength);
+            this.Length = otherLength;
+        }
+
+        /// <summary>
+        /// Appends the given array to this CharsRef
+        /// </summary>
+        public void Append(char[] otherChars, int otherOffset, int otherLength)
+        {
+            int newLen = Length + otherLength;
+            if (this.Chars.Length - this.Offset < newLen)
+            {
+                char[] newChars = new char[newLen];
+                Array.Copy(this.Chars, this.Offset, newChars, 0, Length);
+                this.Offset = 0;
+                this.Chars = newChars;
+            }
+            Array.Copy(otherChars, otherOffset, this.Chars, this.Length + this.Offset, otherLength);
+            Length = newLen;
+        }
+
+        public override string ToString()
+        {
+            return new string(this.Chars, this.Offset, this.Length);
+        }
+
+        public char CharAt(int index)
+        {
+            // NOTE: must do a real check here to meet the specs of CharSequence
+            if (index < 0 || index >= Length)
+            {
+                throw new System.IndexOutOfRangeException();
+            }
+            return this.Chars[this.Offset + index];
+        }
+
+        public ICharSequence SubSequence(int start, int end)
+        {
+            // NOTE: must do a real check here to meet the specs of CharSequence
+            if (start < 0 || end > Length || start > end)
+            {
+                throw new System.IndexOutOfRangeException();
+            }
+            return new CharsRef(this.Chars, this.Offset + start, end - start);
+        }
+
+        /// @deprecated this comparator is only a transition mechanism
+#pragma warning disable 0612, 0618
+        private static readonly IComparer<CharsRef> Utf16SortedAsUTF8SortOrder = new UTF16SortedAsUTF8Comparator();
+#pragma warning restore 0612, 0618
+
+        /// @deprecated this comparator is only a transition mechanism
+        [Obsolete("this comparator is only a transition mechanism")]
+        public static IComparer<CharsRef> UTF16SortedAsUTF8Comparer
+        {
+            get
+            {
+                return Utf16SortedAsUTF8SortOrder;
+            }
+        }
+
+        
+        [Obsolete("this comparator is only a transition mechanism")]
+        private class UTF16SortedAsUTF8Comparator : IComparer<CharsRef>
+        {
+            // Only singleton
+            internal UTF16SortedAsUTF8Comparator()
+            {
+            }
+
+            public virtual int Compare(CharsRef a, CharsRef b)
+            {
+                if (a == b)
+                {
+                    return 0;
+                }
+
+                char[] aChars = a.Chars;
+                int aUpto = a.Offset;
+                char[] bChars = b.Chars;
+                int bUpto = b.Offset;
+
+                int aStop = aUpto + Math.Min(a.Length, b.Length);
+
+                while (aUpto < aStop)
+                {
+                    char aChar = aChars[aUpto++];
+                    char bChar = bChars[bUpto++];
+                    if (aChar != bChar)
+                    {
+                        // http://icu-project.org/docs/papers/utf16_code_point_order.html
+
+                        /* aChar != bChar, fix up each one if they're both in or above the surrogate range, then compare them */
+                        if (aChar >= 0xd800 && bChar >= 0xd800)
+                        {//LUCENE TO-DO possible truncation or is char 16bit?
+                            if (aChar >= 0xe000)
+                            {
+                                aChar -= (char)0x800;
+                            }
+                            else
+                            {
+                                aChar += (char)0x2000;
+                            }
+
+                            if (bChar >= 0xe000)
+                            {
+                                bChar -= (char)0x800;
+                            }
+                            else
+                            {
+                                bChar += (char)0x2000;
+                            }
+                        }
+
+                        /* now aChar and bChar are in code point order */
+                        return (int)aChar - (int)bChar; // int must be 32 bits wide
+                    }
+                }
+
+                // One is a prefix of the other, or, they are equal:
+                return a.Length - b.Length;
+            }
+        }
+
+        /// <summary>
+        /// Creates a new CharsRef that points to a copy of the chars from
+        /// <code>other</code>
+        /// <p>
+        /// The returned CharsRef will have a length of other.length
+        /// and an offset of zero.
+        /// </summary>
+        public static CharsRef DeepCopyOf(CharsRef other)
+        {
+            CharsRef clone = new CharsRef();
+            clone.CopyChars(other);
+            return clone;
+        }
+
+        /// <summary>
+        /// Performs internal consistency checks.
+        /// Always returns true (or throws InvalidOperationException)
+        /// </summary>
+        public bool Valid
+        {
+            get
+            {
+                if (this.Chars == null)
+                {
+                    throw new InvalidOperationException("chars is null");
+                }
+                if (this.Length < 0)
+                {
+                    throw new InvalidOperationException("length is negative: " + Length);
+                }
+                if (this.Length > this.Chars.Length)
+                {
+                    throw new InvalidOperationException("length is out of bounds: " + Length + ",chars.length=" + Chars.Length);
+                }
+                if (this.Offset < 0)
+                {
+                    throw new InvalidOperationException("offset is negative: " + Offset);
+                }
+                if (this.Offset > this.Chars.Length)
+                {
+                    throw new InvalidOperationException("offset out of bounds: " + Offset + ",chars.length=" + Chars.Length);
+                }
+                if (this.Offset + this.Length < 0)
+                {
+                    throw new InvalidOperationException("offset+length is negative: offset=" + Offset + ",length=" + Length);
+                }
+                if (this.Offset + this.Length > this.Chars.Length)
+                {
+                    throw new InvalidOperationException("offset+length out of bounds: offset=" + Offset + ",length=" + Length + ",chars.length=" + Chars.Length);
+                }
+                return true;
+            }
+        }
+
+        public IEnumerator<char> GetEnumerator()
+        {
+            return ((IEnumerable<char>)this.Chars).GetEnumerator();
+        }
+
+        System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator()
+        {
+            return this.Chars.GetEnumerator();
+        }
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/626809a0/src/Lucene.Net.Core/Util/RamUsageEstimator.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Core/Util/RamUsageEstimator.cs b/src/Lucene.Net.Core/Util/RamUsageEstimator.cs
index 4d5e2a4..cc46b66 100644
--- a/src/Lucene.Net.Core/Util/RamUsageEstimator.cs
+++ b/src/Lucene.Net.Core/Util/RamUsageEstimator.cs
@@ -180,34 +180,34 @@ namespace Lucene.Net.Util
 
 
 
-        private static readonly IDictionary<Type, int> primitiveSizes;
+        internal static readonly IDictionary<Type, int> PrimitiveSizes;
 
         static RamUsageEstimator()
         {
-            primitiveSizes = new HashMap<Type, int>();
+            PrimitiveSizes = new HashMap<Type, int>();
 
             // 1 
-            primitiveSizes[typeof(bool)] = NUM_BYTES_BOOLEAN;
-            primitiveSizes[typeof(byte)] = NUM_BYTES_BYTE;
-            primitiveSizes[typeof(sbyte)] = NUM_BYTES_SBYTE;
+            PrimitiveSizes[typeof(bool)] = NUM_BYTES_BOOLEAN;
+            PrimitiveSizes[typeof(byte)] = NUM_BYTES_BYTE;
+            PrimitiveSizes[typeof(sbyte)] = NUM_BYTES_SBYTE;
 
             // 2
-            primitiveSizes[typeof(char)] = NUM_BYTES_CHAR;
-            primitiveSizes[typeof(short)] = NUM_BYTES_SHORT;
-            primitiveSizes[typeof(ushort)] = NUM_BYTES_USHORT;
+            PrimitiveSizes[typeof(char)] = NUM_BYTES_CHAR;
+            PrimitiveSizes[typeof(short)] = NUM_BYTES_SHORT;
+            PrimitiveSizes[typeof(ushort)] = NUM_BYTES_USHORT;
 
             // 4
-            primitiveSizes[typeof(int)] = NUM_BYTES_INT;
-            primitiveSizes[typeof(uint)] = NUM_BYTES_UINT;
-            primitiveSizes[typeof(float)] = NUM_BYTES_FLOAT;
+            PrimitiveSizes[typeof(int)] = NUM_BYTES_INT;
+            PrimitiveSizes[typeof(uint)] = NUM_BYTES_UINT;
+            PrimitiveSizes[typeof(float)] = NUM_BYTES_FLOAT;
 
             // 8
-            primitiveSizes[typeof(long)] = NUM_BYTES_LONG;
-            primitiveSizes[typeof(ulong)] = NUM_BYTES_ULONG;
-            primitiveSizes[typeof(double)] = NUM_BYTES_DOUBLE;
+            PrimitiveSizes[typeof(long)] = NUM_BYTES_LONG;
+            PrimitiveSizes[typeof(ulong)] = NUM_BYTES_ULONG;
+            PrimitiveSizes[typeof(double)] = NUM_BYTES_DOUBLE;
 
             // 16
-            primitiveSizes[typeof(decimal)] = NUM_BYTES_DECIMAL;
+            PrimitiveSizes[typeof(decimal)] = NUM_BYTES_DECIMAL;
 
             // The Java Version references "sun.misc.Unsafe", the closest class to have one or two of the
             // methods that Unsafe has is System.Runtime.InteropServices.Marshal
@@ -253,7 +253,7 @@ namespace Lucene.Net.Util
         public static long AdjustForField(long sizeSoFar, FieldInfo f)
         {
             var typeInfo = f.FieldType.GetTypeInfo();
-            int fsize = typeInfo.IsPrimitive ? primitiveSizes[f.FieldType] : NUM_BYTES_OBJECT_REF;
+            int fsize = typeInfo.IsPrimitive ? PrimitiveSizes[f.FieldType] : NUM_BYTES_OBJECT_REF;
 
             if (f.DeclaringType != null && f.DeclaringType.GetTypeInfo().IsValueType)
             {
@@ -386,7 +386,7 @@ namespace Lucene.Net.Util
                 throw new ArgumentException("This method does not work with Arrays.");
 
             if (typeInfo.IsPrimitive)
-                return primitiveSizes[typeInfo.AsType()];
+                return PrimitiveSizes[typeInfo.AsType()];
 
             long size = NUM_BYTES_OBJECT_HEADER;
 
@@ -412,7 +412,7 @@ namespace Lucene.Net.Util
                 if (arrayElementTypeInfo.IsPrimitive)
                 {
                     size = NUM_BYTES_VALUE_TYPE_ARRAY_HEADER;
-                    size += (long)length * primitiveSizes[arrayElementType];
+                    size += (long)length * PrimitiveSizes[arrayElementType];
                 }
       
                 else
@@ -445,10 +445,10 @@ namespace Lucene.Net.Util
         public static long SizeOf<T>(T[] array) where T : struct
         {
             var type = typeof(T);
-            if (!primitiveSizes.ContainsKey(type))
+            if (!PrimitiveSizes.ContainsKey(type))
                 return ShallowSizeOfArray(array);
 
-            int bytes = primitiveSizes[type];
+            int bytes = PrimitiveSizes[type];
           
             var size = (long)NUM_BYTES_VALUE_TYPE_ARRAY_HEADER + (long)bytes * array.Length;
 

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/626809a0/src/Lucene.Net.Core/Util/StringHelper.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Core/Util/StringHelper.cs b/src/Lucene.Net.Core/Util/StringHelper.cs
new file mode 100644
index 0000000..68122b1
--- /dev/null
+++ b/src/Lucene.Net.Core/Util/StringHelper.cs
@@ -0,0 +1,87 @@
+
+
+namespace Lucene.Net.Util
+{
+    using System;
+    using System.Collections.Generic;
+    using System.Linq;
+    using System.Text;
+    using System.Threading.Tasks;
+    using Lucene.Net.Support;
+
+    public class StringHelper
+    {
+        // WHY is this in string helper?
+        /// <summary>
+        /// Poached from Guava: set a different salt/seed
+        /// for each JVM instance, to frustrate hash key collision
+        /// denial of service attacks, and to catch any places that
+        /// somehow rely on hash function/order across JVM
+        /// instances:
+        /// </summary>
+        public static readonly int GOOD_FAST_HASH_SEED;
+
+        static StringHelper()
+        {
+            var prop = SystemProps.Get("tests:seed");
+
+            if(prop != null)
+            {
+                if(prop.Length > 8)
+                {
+                    prop = prop.Substring(prop.Length - 8);
+                }
+
+                GOOD_FAST_HASH_SEED = (int)int.Parse(prop);
+            } else{
+                GOOD_FAST_HASH_SEED = (int) System.DateTime.Now.Ticks;
+            }
+        }
+
+        public static int MurmurHash3_x86_32(byte[] data, int offset, int length, int seed)
+        {
+            int c1 = unchecked((int)0xcc9e2d51),
+                c2 = 0x1b873593;
+            uint n = 3864292196;
+
+            int h1 = seed;
+            int roundedEnd = offset + (int)(length & 0xfffffffc);  // round down to 4 byte block
+
+            for (int i=offset; i<roundedEnd; i+=4) {
+              // little endian load order
+              int k = (data[i] & 0xff) | ((data[i+1] & 0xff) << 8) | ((data[i+2] & 0xff) << 16) | (data[i+3] << 24);
+              k *= c1;
+              k = k.RotateLeft(15);
+              k *= c2;
+                
+              h1 ^= k;
+              h1 = h1.RotateLeft(13);
+              h1 = h1 * 5+(int)n;
+            }
+
+    // tail
+            int k1 = 0;
+
+            switch(length & 0x03) {
+              case 3:
+                k1 = (data[roundedEnd + 2] & 0xff) << 16;
+                goto case 2;
+              case 2:
+                k1 |= (data[roundedEnd + 1] & 0xff) << 8;
+                goto case 1;
+              case 1:
+                k1 |= (data[roundedEnd] & 0xff);
+                k1 *= c1;
+                k1 = k1.RotateLeft(15);
+                k1 *= c2;
+                h1 ^= k1;
+                break;
+            }
+
+            h1 ^= length;
+
+            return h1.ComputeMurmurHash3();
+        }
+    }
+   
+}

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/626809a0/src/Lucene.Net.Core/Util/UnicodeUtil.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Core/Util/UnicodeUtil.cs b/src/Lucene.Net.Core/Util/UnicodeUtil.cs
new file mode 100644
index 0000000..ef99984
--- /dev/null
+++ b/src/Lucene.Net.Core/Util/UnicodeUtil.cs
@@ -0,0 +1,145 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Lucene.Net.Util
+{
+    public static class UnicodeUtil
+    {
+        public const int UNI_SUR_HIGH_START = 0xD800;
+        public const int UNI_SUR_HIGH_END = 0xDBFF;
+        public const int UNI_SUR_LOW_START = 0xDC00;
+        public const int UNI_SUR_LOW_END = 0xDFFF;
+        public const int UNI_REPLACEMENT_CHAR = 0xFFFD;
+        private const int MIN_SUPPLEMENTARY_CODE_POINT = 0x10000;
+
+        private const long UNI_MAX_BMP = 0x0000FFFF;
+
+        private const long HALF_SHIFT = 10;
+        private const long HALF_MASK = 0x3FFL;
+
+        private static readonly int SURROGATE_OFFSET = MIN_SUPPLEMENTARY_CODE_POINT - (UNI_SUR_HIGH_START << (int)HALF_SHIFT) - UNI_SUR_LOW_START;
+
+        public static void UTF16toUTF8(IEnumerable<char> s, int offset, int length, BytesRef result)
+        {
+            int end = offset + length;
+
+            byte[] @out = result.Bytes;
+            result.Offset = 0;
+            // Pre-allocate for worst case 4-for-1
+            int maxLen = length * 4;
+            if (@out.Length < maxLen)
+            {
+                @out = result.Bytes = new byte[maxLen];
+            }
+
+            int upto = 0;
+            for (int i = offset; i < end; i++)
+            {
+                int code = (int)s.ElementAt(i);
+
+                if (code < 0x80)
+                {
+                    @out[upto++] = (byte)code;
+                }
+                else if (code < 0x800)
+                {
+                    @out[upto++] = unchecked((byte)(0xC0 | (code >> 6)));
+                    @out[upto++] = unchecked((byte)(0x80 | (code & 0x3F)));
+                }
+                else if (code < 0xD800 || code > 0xDFFF)
+                {
+                    @out[upto++] = unchecked((byte)(0xE0 | (code >> 12)));
+                    @out[upto++] = unchecked((byte)(0x80 | ((code >> 6) & 0x3F)));
+                    @out[upto++] = unchecked((byte)(0x80 | (code & 0x3F)));
+                }
+                else
+                {
+                    // surrogate pair
+                    // confirm valid high surrogate
+                    if (code < 0xDC00 && (i < end - 1))
+                    {
+                        int utf32 = (int)s.ElementAt(i + 1);
+                        // confirm valid low surrogate and write pair
+                        if (utf32 >= 0xDC00 && utf32 <= 0xDFFF)
+                        {
+                            utf32 = (code << 10) + utf32 + SURROGATE_OFFSET;
+                            i++;
+                            @out[upto++] = unchecked((byte)(0xF0 | (utf32 >> 18)));
+                            @out[upto++] = unchecked((byte)(0x80 | ((utf32 >> 12) & 0x3F)));
+                            @out[upto++] = unchecked((byte)(0x80 | ((utf32 >> 6) & 0x3F)));
+                            @out[upto++] = unchecked((byte)(0x80 | (utf32 & 0x3F)));
+                            continue;
+                        }
+                    }
+                    // replace unpaired surrogate or out-of-order low surrogate
+                    // with substitution character
+                    @out[upto++] = unchecked((byte)0xEF);
+                    @out[upto++] = unchecked((byte)0xBF);
+                    @out[upto++] = unchecked((byte)0xBD);
+                }
+            }
+            //assert matches(s, offset, length, out, upto);
+            result.Length = upto;
+        }
+
+
+
+        /// <summary>
+        /// Interprets the given byte array as UTF-8 and converts to UTF-16. The <seealso cref="CharsRef"/> will be extended if
+        /// it doesn't provide enough space to hold the worst case of each byte becoming a UTF-16 codepoint.
+        /// <p>
+        /// NOTE: Full characters are read, even if this reads past the length passed (and
+        /// can result in an ArrayOutOfBoundsException if invalid UTF-8 is passed).
+        /// Explicit checks for valid UTF-8 are not performed.
+        /// </summary>
+        // TODO: broken if chars.offset != 0
+        public static void UTF8toUTF16(byte[] utf8, int offset, int length, CharsRef chars)
+        {
+            int outOffset = chars.Offset = 0,
+                limit = offset + length;
+
+            char[] @out = chars.Chars = ArrayUtil.Grow(chars.Chars, length);
+  
+            while (offset < limit)
+            {
+                int b = utf8[offset++] & 0xff;
+                if (b < 0xc0)
+                {
+                    Debug.Assert(b < 0x80);
+                    @out[outOffset++] = (char)b;
+                }
+                else if (b < 0xe0)
+                {
+                    @out[outOffset++] = (char)(((b & 0x1f) << 6) + (utf8[offset++] & 0x3f));
+                }
+                else if (b < 0xf0)
+                {
+                    @out[outOffset++] = (char)(((b & 0xf) << 12) + ((utf8[offset] & 0x3f) << 6) + (utf8[offset + 1] & 0x3f));
+                    offset += 2;
+                }
+                else
+                {
+                    Debug.Assert(b < 0xf8, "b = 0x" + b.ToString("x"));
+
+                    int ch = ((b & 0x7) << 18) + ((utf8[offset] & 0x3f) << 12) + ((utf8[offset + 1] & 0x3f) << 6) + (utf8[offset + 2] & 0x3f);
+                    offset += 3;
+                    if (ch < UNI_MAX_BMP)
+                    {
+                        @out[outOffset++] = (char)ch;
+                    }
+                    else
+                    {
+                        int chHalf = ch - 0x0010000;
+                        @out[outOffset++] = (char)((chHalf >> 10) + 0xD800);
+                        @out[outOffset++] = (char)((chHalf & HALF_MASK) + 0xDC00);
+                    }
+                }
+            }
+            chars.Length = outOffset - chars.Offset;
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/626809a0/test/Lucene.Net.Core.Tests/Lucene.Net.Core.Tests.csproj
----------------------------------------------------------------------
diff --git a/test/Lucene.Net.Core.Tests/Lucene.Net.Core.Tests.csproj b/test/Lucene.Net.Core.Tests/Lucene.Net.Core.Tests.csproj
index ec3e52c..6a44bcd 100644
--- a/test/Lucene.Net.Core.Tests/Lucene.Net.Core.Tests.csproj
+++ b/test/Lucene.Net.Core.Tests/Lucene.Net.Core.Tests.csproj
@@ -63,6 +63,8 @@
     <Compile Include="Util\TestArrayUtil.cs" />
     <Compile Include="Util\TestBits.cs" />
     <Compile Include="Util\TestBroadWord.cs" />
+    <Compile Include="Util\TestBytesRef.cs" />
+    <Compile Include="Util\TestCharsRef.cs" />
     <Compile Include="Util\TestConstants.cs" />
     <Compile Include="Util\TestInPlaceMergeSorter.cs" />
     <Compile Include="Util\TestRamEstimatorUsage.cs" />

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/626809a0/test/Lucene.Net.Core.Tests/Util/BaseSorterTestCase.cs
----------------------------------------------------------------------
diff --git a/test/Lucene.Net.Core.Tests/Util/BaseSorterTestCase.cs b/test/Lucene.Net.Core.Tests/Util/BaseSorterTestCase.cs
index 693acac..5d57069 100644
--- a/test/Lucene.Net.Core.Tests/Util/BaseSorterTestCase.cs
+++ b/test/Lucene.Net.Core.Tests/Util/BaseSorterTestCase.cs
@@ -22,8 +22,7 @@ namespace Lucene.Net.Util
     using System;
     using System.Linq;
     using System.Collections.Generic;
-    using Lucene.Net.TestFramework;
-    using Lucene.Net.TestFramework.Random;
+    using Lucene.Net.Random;
 
     public abstract class BaseSorterTestCase : LuceneTestCase
     {

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/626809a0/test/Lucene.Net.Core.Tests/Util/TestBits.cs
----------------------------------------------------------------------
diff --git a/test/Lucene.Net.Core.Tests/Util/TestBits.cs b/test/Lucene.Net.Core.Tests/Util/TestBits.cs
index c47f701..650dd53 100644
--- a/test/Lucene.Net.Core.Tests/Util/TestBits.cs
+++ b/test/Lucene.Net.Core.Tests/Util/TestBits.cs
@@ -18,7 +18,6 @@
 namespace Lucene.Net.Util
 {
     using System;
-    using Lucene.Net.TestFramework;
 
     /// <summary>
     /// Test Bits

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/626809a0/test/Lucene.Net.Core.Tests/Util/TestBytesRef.cs
----------------------------------------------------------------------
diff --git a/test/Lucene.Net.Core.Tests/Util/TestBytesRef.cs b/test/Lucene.Net.Core.Tests/Util/TestBytesRef.cs
new file mode 100644
index 0000000..5ef6ba3
--- /dev/null
+++ b/test/Lucene.Net.Core.Tests/Util/TestBytesRef.cs
@@ -0,0 +1,49 @@
+
+
+namespace Lucene.Net.Util
+{
+    public class TestBytesRef : LuceneTestCase
+    {
+        [Test]
+        public void TestEmpty()
+        {
+            var b = new BytesRef();
+            Equal(BytesRef.EMPTY_BYTES, b.Bytes);
+            Equal(0, b.Offset);
+            Equal(0, b.Length);
+        }
+
+        [Test]
+        public void TestFromBytes()
+        {
+
+            byte[] bytes = new byte[] { (byte)'a', (byte)'b', (byte)'c', (byte)'d' };
+            var bytesRef = new BytesRef(bytes);
+            Equal(bytes, bytesRef.Bytes);
+            Equal(0, bytesRef.Offset);
+            Equal(bytes.Length, bytesRef.Length);
+
+            var bytesRef2 = new BytesRef(bytes, 1, 3);
+            Equal("bcd", bytesRef2.Utf8ToString());
+
+            Ok(!bytesRef.Equals(bytesRef2), "bytesRef should not equal bytesRef2");
+        }
+
+
+        [Test]
+        public void TestFromChars()
+        {
+            100.Times((i) =>
+            {
+                var utf8str1 = this.Random.ToUnicodeString();
+                var utf8str2 = new BytesRef(utf8str1).Utf8ToString();
+                Equal(utf8str1, utf8str2);
+            });
+            var value = "\uFFFF";
+
+            Equal(value, new BytesRef(value).Utf8ToString());
+        }
+
+
+    }
+}

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/626809a0/test/Lucene.Net.Core.Tests/Util/TestCharsRef.cs
----------------------------------------------------------------------
diff --git a/test/Lucene.Net.Core.Tests/Util/TestCharsRef.cs b/test/Lucene.Net.Core.Tests/Util/TestCharsRef.cs
new file mode 100644
index 0000000..052abc5
--- /dev/null
+++ b/test/Lucene.Net.Core.Tests/Util/TestCharsRef.cs
@@ -0,0 +1,25 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+namespace Lucene.Net.Util
+{
+    using System;
+
+    public class TestCharsRef
+    {
+    }
+}

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/626809a0/test/Lucene.Net.Core.Tests/Util/TestConstants.cs
----------------------------------------------------------------------
diff --git a/test/Lucene.Net.Core.Tests/Util/TestConstants.cs b/test/Lucene.Net.Core.Tests/Util/TestConstants.cs
index 8a3c583..93f9eb4 100644
--- a/test/Lucene.Net.Core.Tests/Util/TestConstants.cs
+++ b/test/Lucene.Net.Core.Tests/Util/TestConstants.cs
@@ -21,7 +21,6 @@ namespace Lucene.Net.Util
     using System;
     using System.Text.RegularExpressions;
     using Lucene.Net.Support;
-    using Lucene.Net.TestFramework;
 
 
     [System.CLSCompliant(false)]

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/626809a0/test/Lucene.Net.Core.Tests/Util/TestRamEstimatorUsage.cs
----------------------------------------------------------------------
diff --git a/test/Lucene.Net.Core.Tests/Util/TestRamEstimatorUsage.cs b/test/Lucene.Net.Core.Tests/Util/TestRamEstimatorUsage.cs
index bc17c14..d498c35 100644
--- a/test/Lucene.Net.Core.Tests/Util/TestRamEstimatorUsage.cs
+++ b/test/Lucene.Net.Core.Tests/Util/TestRamEstimatorUsage.cs
@@ -20,7 +20,6 @@
 namespace Lucene.Net.Util
 {
     using System;
-    using Lucene.Net.TestFramework;
     using System.Diagnostics.CodeAnalysis;
 
     public class TestRamUsageEstimator : LuceneTestCase

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/626809a0/test/Lucene.Net.Core.Tests/Util/TestVersion.cs
----------------------------------------------------------------------
diff --git a/test/Lucene.Net.Core.Tests/Util/TestVersion.cs b/test/Lucene.Net.Core.Tests/Util/TestVersion.cs
index 793479c..f4bd081 100644
--- a/test/Lucene.Net.Core.Tests/Util/TestVersion.cs
+++ b/test/Lucene.Net.Core.Tests/Util/TestVersion.cs
@@ -21,7 +21,6 @@
 
 namespace Lucene.Net.Util
 {
-    using Lucene.Net.TestFramework;
     using System;
     using System.Linq;
     using System.Reflection;

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/626809a0/test/Lucene.Net.TestFramework/Lucene.Net.TestFramework.csproj
----------------------------------------------------------------------
diff --git a/test/Lucene.Net.TestFramework/Lucene.Net.TestFramework.csproj b/test/Lucene.Net.TestFramework/Lucene.Net.TestFramework.csproj
index 32f35f1..b44dcff 100644
--- a/test/Lucene.Net.TestFramework/Lucene.Net.TestFramework.csproj
+++ b/test/Lucene.Net.TestFramework/Lucene.Net.TestFramework.csproj
@@ -57,11 +57,13 @@
     <WarningLevel>4</WarningLevel>
   </PropertyGroup>
   <ItemGroup>
+    <Compile Include="Util\ExtensionMethods.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
     <Compile Include="Random\RandomExtensions.cs" />
     <Compile Include="Util\LuceneTestCase.cs" />
     <Compile Include="Util\RamUsageTester.cs" />
     <Compile Include="Util\TestAttribute.cs" />
+    <Compile Include="Util\TestUtil.cs" />
   </ItemGroup>
   <ItemGroup>
     <Reference Include="xunit.abstractions">

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/626809a0/test/Lucene.Net.TestFramework/Random/RandomExtensions.cs
----------------------------------------------------------------------
diff --git a/test/Lucene.Net.TestFramework/Random/RandomExtensions.cs b/test/Lucene.Net.TestFramework/Random/RandomExtensions.cs
index 78242fb..48ebea8 100644
--- a/test/Lucene.Net.TestFramework/Random/RandomExtensions.cs
+++ b/test/Lucene.Net.TestFramework/Random/RandomExtensions.cs
@@ -16,7 +16,7 @@
  */
 
 
-namespace Lucene.Net.TestFramework.Random
+namespace Lucene.Net.Random
 {
     using System;
     using System.Diagnostics;
@@ -31,7 +31,20 @@ namespace Lucene.Net.TestFramework.Random
     /// </remarks>
     public static class RandomExtensions
     {
-	     
+	    /// <summary>
+	    /// Returns an integer between the min and max value. This is compatable 
+        /// with the Java Lucene version of the NextIntBetween method.
+	    /// </summary>
+        /// <remarks>
+        ///    <para>
+        ///        .NET has a default overloade for <see cref="System.Random.Next"/> that has a min and
+        ///        max value. However, this method exists to keep capatablity with the Java Version of Lucene.
+        ///    </para>
+        /// </remarks>
+	    /// <param name="random">The instance of random.</param>
+        /// <param name="minValue">The minimum value that the random may use.</param>
+        /// <param name="maxValue">The maximum value that the random may use.</param>
+	    /// <returns>A random integer.</returns>
         public static int NextBetween(this Random random, int minValue, int maxValue)
         {
             Debug.Assert(maxValue >= minValue, string.Format("maxValue must be greater than minValue" +

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/626809a0/test/Lucene.Net.TestFramework/Util/ExtensionMethods.cs
----------------------------------------------------------------------
diff --git a/test/Lucene.Net.TestFramework/Util/ExtensionMethods.cs b/test/Lucene.Net.TestFramework/Util/ExtensionMethods.cs
new file mode 100644
index 0000000..8b60178
--- /dev/null
+++ b/test/Lucene.Net.TestFramework/Util/ExtensionMethods.cs
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+namespace Lucene.Net.Util
+{
+    using System;
+
+    /// <summary>
+    /// Extension methods to make writing tests easier.
+    /// </summary>
+    public static class ExtensionMethods
+    {
+        /// <summary>
+        /// Performs a loop for the specified number of times. 
+        /// </summary>
+        /// <param name="value">The number of times to perform a loop.</param>
+        /// <param name="invoke">The code that should be ivnoked for each iteration.</param>
+        public static void Times(this int value, Action<int> invoke)
+        {
+            for(var i = 0; i < value; i++)
+            {
+                invoke(i);
+            }
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/626809a0/test/Lucene.Net.TestFramework/Util/LuceneTestCase.cs
----------------------------------------------------------------------
diff --git a/test/Lucene.Net.TestFramework/Util/LuceneTestCase.cs b/test/Lucene.Net.TestFramework/Util/LuceneTestCase.cs
index d0045fb..3c08e61 100644
--- a/test/Lucene.Net.TestFramework/Util/LuceneTestCase.cs
+++ b/test/Lucene.Net.TestFramework/Util/LuceneTestCase.cs
@@ -96,6 +96,11 @@ namespace Lucene.Net.Util
             Assert.Equal(expected, actual);
         }
 
+        [DebuggerHidden]
+        public static void NotEqual<T>(T expected, T actual)
+        {
+            Assert.NotEqual(expected, actual);
+        }
 
 
         [DebuggerHidden]

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/626809a0/test/Lucene.Net.TestFramework/Util/TestUtil.cs
----------------------------------------------------------------------
diff --git a/test/Lucene.Net.TestFramework/Util/TestUtil.cs b/test/Lucene.Net.TestFramework/Util/TestUtil.cs
new file mode 100644
index 0000000..c161fb9
--- /dev/null
+++ b/test/Lucene.Net.TestFramework/Util/TestUtil.cs
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+namespace Lucene.Net.Util
+{
+    using System;
+    using System.Collections.Generic;
+    using System.Linq;
+    using System.Text;
+    using System.Threading.Tasks;
+    using Lucene.Net.Random;
+
+
+    public static class  TestUtil
+    {
+
+        /// <summary>
+        /// Returns a random string up to the specified length.
+        /// </summary>
+        /// <param name="random">The random instance used to generate characters.</param>
+        /// <param name="maxLength">The maximum length for the generated string.</param>
+        /// <returns>A random string.</returns>
+        public static string ToUnicodeString(this Random random, int maxLength = 20)
+        {
+            int end = random.NextBetween(0, maxLength);
+            if (end == 0)
+                return "";
+
+            var buffer = new char[end];
+            random.RandomFixedLengthUnicodeString(buffer, 0, buffer.Length);
+
+            return new String(buffer, 0, end);
+        }
+
+        /// <summary>
+        /// Fills <paramref name="chars"/> with a valid random unicode character sequence.
+        /// </summary>
+        /// <param name="random">The random instance used to generate characters.</param>
+        /// <param name="chars">The character array that will be filled.</param>
+        /// <param name="offset">The position to start the character fill.</param>
+        /// <param name="length">The number of characters that should be generated.</param>
+        public static void RandomFixedLengthUnicodeString(this Random random, char[] chars, int offset, int length)
+        {
+            int i = offset,
+                end = offset + length;
+
+            while(i < end)
+            {
+                int t = random.Next(5);
+                if(0 == t && i < length - 1)
+                {
+                    // Make a surrogate pair
+                    // High surrogate
+                    chars[i++] = (char)random.NextBetween(0xd800, 0xdbff);
+                    // low surrogate
+                    chars[i++] = (char)random.NextBetween(0xdc00, 0xdfff);
+                }
+                else if (t <= 1)
+                {
+                    chars[i++] = (char)random.Next(0x80);
+                }
+                else if (t == 2)
+                {
+                    chars[i++] = (char)random.Next(0x80, 0x7ff);
+                }
+                else if (t == 3)
+                {
+                    chars[i++] = (char)random.Next(0x800, 0xd7ff);
+                }
+                else if (t == 4)
+                {
+                    chars[i++] = (char)random.Next(0xe000, 0xffff);
+                }
+            }
+        }
+    }
+}


Mime
View raw message