Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id B5D1D200BB9 for ; Sun, 2 Oct 2016 12:16:34 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id B332A160AF7; Sun, 2 Oct 2016 10:16:34 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id A48D8160AEA for ; Sun, 2 Oct 2016 12:16:33 +0200 (CEST) Received: (qmail 72998 invoked by uid 500); 2 Oct 2016 10:16:32 -0000 Mailing-List: contact commits-help@lucenenet.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: lucene-net-dev@lucenenet.apache.org Delivered-To: mailing list commits@lucenenet.apache.org Received: (qmail 72887 invoked by uid 99); 2 Oct 2016 10:16:32 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sun, 02 Oct 2016 10:16:32 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 875E1E0A3F; Sun, 2 Oct 2016 10:16:32 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: synhershko@apache.org To: commits@lucenenet.apache.org Date: Sun, 02 Oct 2016 10:16:34 -0000 Message-Id: In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [03/49] lucenenet git commit: Fixed bugs in FST that were causing test failures and Debug.Assert failures in Misc.Util.Fst. archived-at: Sun, 02 Oct 2016 10:16:34 -0000 Fixed bugs in FST that were causing test failures and Debug.Assert failures in Misc.Util.Fst. Project: http://git-wip-us.apache.org/repos/asf/lucenenet/repo Commit: http://git-wip-us.apache.org/repos/asf/lucenenet/commit/243ada71 Tree: http://git-wip-us.apache.org/repos/asf/lucenenet/tree/243ada71 Diff: http://git-wip-us.apache.org/repos/asf/lucenenet/diff/243ada71 Branch: refs/heads/master Commit: 243ada716c47dafeaf8d2aa39c9ebd8f4c0e8bd7 Parents: b56ebc2 Author: Shad Storhaug Authored: Mon Sep 5 18:21:23 2016 +0700 Committer: Shad Storhaug Committed: Wed Sep 7 17:34:51 2016 +0700 ---------------------------------------------------------------------- src/Lucene.Net.Core/Util/Fst/BytesStore.cs | 18 ++--- src/Lucene.Net.Core/Util/Fst/FST.cs | 18 ++++- src/Lucene.Net.Core/Util/Fst/NodeHash.cs | 80 +++++++++++++++----- .../Util/fst/FSTTester.cs | 24 +++++- 4 files changed, 110 insertions(+), 30 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lucenenet/blob/243ada71/src/Lucene.Net.Core/Util/Fst/BytesStore.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Core/Util/Fst/BytesStore.cs b/src/Lucene.Net.Core/Util/Fst/BytesStore.cs index de949b8..471cd68 100644 --- a/src/Lucene.Net.Core/Util/Fst/BytesStore.cs +++ b/src/Lucene.Net.Core/Util/Fst/BytesStore.cs @@ -375,7 +375,7 @@ namespace Lucene.Net.Util.Fst blockIndex--; NextWrite = BlockSize; } - Blocks.GetRange(blockIndex + 1, Blocks.Count).Clear(); + Blocks.RemoveRange(blockIndex + 1, Blocks.Count - (blockIndex + 1)); if (newLen == 0) { Current = null; @@ -430,7 +430,7 @@ namespace Lucene.Net.Util.Fst nextRead = outerInstance.BlockSize; } - private sbyte[] Current; + private byte[] Current; private int nextBuffer; private int nextRead; @@ -438,15 +438,15 @@ namespace Lucene.Net.Util.Fst { if (nextRead == OuterInstance.BlockSize) { - OuterInstance.Current = OuterInstance.Blocks[nextBuffer++]; + Current = OuterInstance.Blocks[nextBuffer++]; nextRead = 0; } - return OuterInstance.Current[nextRead++]; + return Current[nextRead++]; } public override void SkipBytes(int count) { - Position = OuterInstance.Position + count; + Position = Position + count; } public override void ReadBytes(byte[] b, int offset, int len) @@ -456,7 +456,7 @@ namespace Lucene.Net.Util.Fst int chunkLeft = OuterInstance.BlockSize - nextRead; if (len <= chunkLeft) { - Array.Copy(OuterInstance.Current, nextRead, b, offset, len); + Array.Copy(Current, nextRead, b, offset, len); nextRead += len; break; } @@ -464,11 +464,11 @@ namespace Lucene.Net.Util.Fst { if (chunkLeft > 0) { - Array.Copy(OuterInstance.Current, nextRead, b, offset, chunkLeft); + Array.Copy(Current, nextRead, b, offset, chunkLeft); offset += chunkLeft; len -= chunkLeft; } - OuterInstance.Current = OuterInstance.Blocks[nextBuffer++]; + Current = OuterInstance.Blocks[nextBuffer++]; nextRead = 0; } } @@ -484,7 +484,7 @@ namespace Lucene.Net.Util.Fst { int bufferIndex = (int)(value >> OuterInstance.blockBits); nextBuffer = bufferIndex + 1; - OuterInstance.Current = OuterInstance.Blocks[bufferIndex]; + Current = OuterInstance.Blocks[bufferIndex]; nextRead = (int)(value & OuterInstance.BlockMask); Debug.Assert(this.Position == value, "pos=" + value + " getPos()=" + this.Position); } http://git-wip-us.apache.org/repos/asf/lucenenet/blob/243ada71/src/Lucene.Net.Core/Util/Fst/FST.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Core/Util/Fst/FST.cs b/src/Lucene.Net.Core/Util/Fst/FST.cs index d9dd78c..74beb96 100644 --- a/src/Lucene.Net.Core/Util/Fst/FST.cs +++ b/src/Lucene.Net.Core/Util/Fst/FST.cs @@ -6,6 +6,7 @@ using System.Text; namespace Lucene.Net.Util.Fst { using Lucene.Net.Util; + using System.Collections; using System.IO; using ByteArrayDataOutput = Lucene.Net.Store.ByteArrayDataOutput; @@ -439,7 +440,22 @@ namespace Lucene.Net.Util.Fst Debug.Assert(root.Flags == asserting.Flags); Debug.Assert(root.Label == asserting.Label); Debug.Assert(root.NextArc == asserting.NextArc); - Debug.Assert(root.NextFinalOutput.Equals(asserting.NextFinalOutput)); + // LUCENENET NOTE: In .NET, IEnumerable will not equal another identical IEnumerable + // because it checks for reference equality, not that the list contents + // are the same. + if (root.NextFinalOutput is IEnumerable && asserting.NextFinalOutput is IEnumerable) + { + var iter = (asserting.NextFinalOutput as IEnumerable).GetEnumerator(); + foreach (object value in root.NextFinalOutput as IEnumerable) + { + iter.MoveNext(); + Debug.Assert(object.Equals(value, iter.Current)); + } + } + else + { + Debug.Assert(root.NextFinalOutput.Equals(asserting.NextFinalOutput)); + } Debug.Assert(root.Node == asserting.Node); Debug.Assert(root.NumArcs == asserting.NumArcs); Debug.Assert(root.Output.Equals(asserting.Output)); http://git-wip-us.apache.org/repos/asf/lucenenet/blob/243ada71/src/Lucene.Net.Core/Util/Fst/NodeHash.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Core/Util/Fst/NodeHash.cs b/src/Lucene.Net.Core/Util/Fst/NodeHash.cs index 4e8bc73..ad8f2d3 100644 --- a/src/Lucene.Net.Core/Util/Fst/NodeHash.cs +++ b/src/Lucene.Net.Core/Util/Fst/NodeHash.cs @@ -2,22 +2,23 @@ using System.Diagnostics; namespace Lucene.Net.Util.Fst { + using System.Collections; /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ using PackedInts = Lucene.Net.Util.Packed.PackedInts; using PagedGrowableWriter = Lucene.Net.Util.Packed.PagedGrowableWriter; @@ -87,7 +88,29 @@ namespace Lucene.Net.Util.Fst long n = ((Builder.CompiledNode)arc.Target).Node; h = PRIME * h + (int)(n ^ (n >> 32)); h = PRIME * h + arc.Output.GetHashCode(); - h = PRIME * h + arc.NextFinalOutput.GetHashCode(); + + // LUCENENET: Since lists do not compare values by default in .NET, + // we need this workaround to get the hashcode of the type + all of the + // values. + if (arc.NextFinalOutput is IEnumerable) + { + h = PRIME * h + arc.NextFinalOutput.GetType().GetHashCode(); + foreach (object value in arc.NextFinalOutput as IEnumerable) + { + if (value != null) + { + h = PRIME * h + value.GetHashCode(); + } + else + { + h = PRIME * h + 0; // 0 for null + } + } + } + else + { + h = PRIME * h + arc.NextFinalOutput.GetHashCode(); + } if (arc.IsFinal) { h += 17; @@ -110,7 +133,30 @@ namespace Lucene.Net.Util.Fst h = PRIME * h + ScratchArc.Label; h = PRIME * h + (int)(ScratchArc.Target ^ (ScratchArc.Target >> 32)); h = PRIME * h + ScratchArc.Output.GetHashCode(); - h = PRIME * h + ScratchArc.NextFinalOutput.GetHashCode(); + + // LUCENENET: Since lists do not compare values by default in .NET, + // we need this workaround to get the hashcode of the type + all of the + // values. + if (ScratchArc.NextFinalOutput is IEnumerable) + { + h = PRIME * h + ScratchArc.NextFinalOutput.GetType().GetHashCode(); + foreach (object value in ScratchArc.NextFinalOutput as IEnumerable) + { + if (value != null) + { + h = PRIME * h + value.GetHashCode(); + } + else + { + h = PRIME * h + 0; // 0 for null + } + } + } + else + { + h = PRIME * h + ScratchArc.NextFinalOutput.GetHashCode(); + } + if (ScratchArc.Final) { h += 17; http://git-wip-us.apache.org/repos/asf/lucenenet/blob/243ada71/src/Lucene.Net.TestFramework/Util/fst/FSTTester.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.TestFramework/Util/fst/FSTTester.cs b/src/Lucene.Net.TestFramework/Util/fst/FSTTester.cs index 9a72d60..d86b695 100644 --- a/src/Lucene.Net.TestFramework/Util/fst/FSTTester.cs +++ b/src/Lucene.Net.TestFramework/Util/fst/FSTTester.cs @@ -327,11 +327,11 @@ namespace Lucene.Net.Util.Fst foreach (InputOutput pair in Pairs) { - if (pair.Output is IList) + if (pair.Output is IEnumerable) { - IList longValues = (IList)pair.Output; Builder builderObject = builder as Builder; - foreach (long value in longValues) + var values = pair.Output as IEnumerable; + foreach (object value in values) { builderObject.Add(pair.Input, value); } @@ -395,6 +395,24 @@ namespace Lucene.Net.Util.Fst protected internal virtual bool OutputsEqual(T a, T b) { + // LUCENENET: In .NET, lists do not automatically test to ensure + // their values are equal, so we need to do that manually. + // Note that we are testing the values without regard to whether + // the enumerable type is nullable. + if (a is IEnumerable && b is IEnumerable) + { + var iter = (b as IEnumerable).GetEnumerator(); + foreach (object value in a as IEnumerable) + { + iter.MoveNext(); + if (!object.Equals(value, iter.Current)) + { + return false; + } + } + return true; + } + return a.Equals(b); }