lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r1688419 - in /lucene/dev/branches/branch_5x: ./ lucene/ lucene/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/ja/util/ lucene/backward-codecs/src/java/org/apache/lucene/codecs/blocktree/ lucene/core/ lucene/core/src/java/org/a...
Date Tue, 30 Jun 2015 10:07:38 GMT
Author: mikemccand
Date: Tue Jun 30 10:07:37 2015
New Revision: 1688419

URL: http://svn.apache.org/r1688419
Log:
LUCENE-6617: reduce heap usage for small FSTs

Modified:
    lucene/dev/branches/branch_5x/   (props changed)
    lucene/dev/branches/branch_5x/lucene/   (props changed)
    lucene/dev/branches/branch_5x/lucene/CHANGES.txt   (contents, props changed)
    lucene/dev/branches/branch_5x/lucene/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/ja/util/TokenInfoDictionaryBuilder.java
    lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/blocktree/Lucene40SegmentTermsEnum.java
    lucene/dev/branches/branch_5x/lucene/core/   (props changed)
    lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/blocktree/SegmentTermsEnum.java
    lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/blocktree/Stats.java
    lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/util/fst/Builder.java
    lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/util/fst/FST.java
    lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/util/fst/NodeHash.java
    lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/util/fst/Test2BFST.java
    lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java
    lucene/dev/branches/branch_5x/lucene/test-framework/   (props changed)
    lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/util/fst/FSTTester.java

Modified: lucene/dev/branches/branch_5x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/CHANGES.txt?rev=1688419&r1=1688418&r2=1688419&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_5x/lucene/CHANGES.txt Tue Jun 30 10:07:37 2015
@@ -210,6 +210,8 @@ Optimizations
   number, by using an array instead of TreeMap except in very sparse
   cases (Robert Muir, Mike McCandless)
 
+* LUCENE-6617: Reduce heap usage for small FSTs (Mike McCandless)
+
 Build
 
 * LUCENE-6518: Don't report false thread leaks from IBM J9

Modified: lucene/dev/branches/branch_5x/lucene/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/ja/util/TokenInfoDictionaryBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/ja/util/TokenInfoDictionaryBuilder.java?rev=1688419&r1=1688418&r2=1688419&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/ja/util/TokenInfoDictionaryBuilder.java
(original)
+++ lucene/dev/branches/branch_5x/lucene/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/ja/util/TokenInfoDictionaryBuilder.java
Tue Jun 30 10:07:37 2015
@@ -165,7 +165,7 @@ public class TokenInfoDictionaryBuilder
     
     final FST<Long> fst = fstBuilder.finish();
     
-    System.out.print("  " + fst.getNodeCount() + " nodes, " + fst.getArcCount() + " arcs,
" + fst.ramBytesUsed() + " bytes...  ");
+    System.out.print("  " + fstBuilder.getNodeCount() + " nodes, " + fstBuilder.getArcCount()
+ " arcs, " + fst.ramBytesUsed() + " bytes...  ");
     dictionary.setFST(fst);
     System.out.println(" done");
     

Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/blocktree/Lucene40SegmentTermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/blocktree/Lucene40SegmentTermsEnum.java?rev=1688419&r1=1688418&r2=1688419&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/blocktree/Lucene40SegmentTermsEnum.java
(original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/blocktree/Lucene40SegmentTermsEnum.java
Tue Jun 30 10:07:37 2015
@@ -127,8 +127,6 @@ final class Lucene40SegmentTermsEnum ext
 
     Lucene40Stats stats = new Lucene40Stats(fr.parent.segment, fr.fieldInfo.name);
     if (fr.index != null) {
-      stats.indexNodeCount = fr.index.getNodeCount();
-      stats.indexArcCount = fr.index.getArcCount();
       stats.indexNumBytes = fr.index.ramBytesUsed();
     }
         

Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/blocktree/SegmentTermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/blocktree/SegmentTermsEnum.java?rev=1688419&r1=1688418&r2=1688419&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/blocktree/SegmentTermsEnum.java
(original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/blocktree/SegmentTermsEnum.java
Tue Jun 30 10:07:37 2015
@@ -124,8 +124,6 @@ final class SegmentTermsEnum extends Ter
 
     Stats stats = new Stats(fr.parent.segment, fr.fieldInfo.name);
     if (fr.index != null) {
-      stats.indexNodeCount = fr.index.getNodeCount();
-      stats.indexArcCount = fr.index.getArcCount();
       stats.indexNumBytes = fr.index.ramBytesUsed();
     }
         

Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/blocktree/Stats.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/blocktree/Stats.java?rev=1688419&r1=1688418&r2=1688419&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/blocktree/Stats.java
(original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/blocktree/Stats.java
Tue Jun 30 10:07:37 2015
@@ -33,12 +33,6 @@ import org.apache.lucene.util.IOUtils;
  * @lucene.internal
  */
 public class Stats {
-  /** How many nodes in the index FST. */
-  public long indexNodeCount;
-
-  /** How many arcs in the index FST. */
-  public long indexArcCount;
-
   /** Byte size of the index. */
   public long indexNumBytes;
 
@@ -163,8 +157,6 @@ public class Stats {
     }
       
     out.println("  index FST:");
-    out.println("    " + indexNodeCount + " nodes");
-    out.println("    " + indexArcCount + " arcs");
     out.println("    " + indexNumBytes + " bytes");
     out.println("  terms:");
     out.println("    " + totalTermCount + " terms");

Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/util/fst/Builder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/util/fst/Builder.java?rev=1688419&r1=1688418&r2=1688419&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/util/fst/Builder.java
(original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/util/fst/Builder.java
Tue Jun 30 10:07:37 2015
@@ -52,7 +52,7 @@ import org.apache.lucene.util.packed.Pac
 
 public class Builder<T> {
   private final NodeHash<T> dedupHash;
-  private final FST<T> fst;
+  final FST<T> fst;
   private final T NO_OUTPUT;
 
   // private static final boolean DEBUG = true;
@@ -81,6 +81,22 @@ public class Builder<T> {
   // current "frontier"
   private UnCompiledNode<T>[] frontier;
 
+  // Used for the BIT_TARGET_NEXT optimization (whereby
+  // instead of storing the address of the target node for
+  // a given arc, we mark a single bit noting that the next
+  // node in the byte[] is the target node):
+  long lastFrozenNode;
+
+  // Reused temporarily while building the FST:
+  int[] reusedBytesPerArc = new int[4];
+
+  long arcCount;
+  long nodeCount;
+
+  boolean allowArrayArcs;
+
+  BytesStore bytes;
+
   /**
    * Instantiates an FST/FSA builder without any pruning. A shortcut
    * to {@link #Builder(FST.INPUT_TYPE, int, int, boolean,
@@ -152,9 +168,12 @@ public class Builder<T> {
     this.shareMaxTailLength = shareMaxTailLength;
     this.doPackFST = doPackFST;
     this.acceptableOverheadRatio = acceptableOverheadRatio;
-    fst = new FST<>(inputType, outputs, doPackFST, acceptableOverheadRatio, allowArrayArcs,
bytesPageBits);
+    this.allowArrayArcs = allowArrayArcs;
+    fst = new FST<>(inputType, outputs, doPackFST, acceptableOverheadRatio, bytesPageBits);
+    bytes = fst.bytes;
+    assert bytes != null;
     if (doShareSuffix) {
-      dedupHash = new NodeHash<>(fst, fst.bytes.getReverseReader(false));
+      dedupHash = new NodeHash<>(fst, bytes.getReverseReader(false));
     } else {
       dedupHash = null;
     }
@@ -168,31 +187,45 @@ public class Builder<T> {
     }
   }
 
-  public long getTotStateCount() {
-    return fst.nodeCount;
-  }
-
   public long getTermCount() {
     return frontier[0].inputCount;
   }
 
+  public long getNodeCount() {
+    // 1+ in order to count the -1 implicit final node
+    return 1+nodeCount;
+  }
+  
+  public long getArcCount() {
+    return arcCount;
+  }
+
   public long getMappedStateCount() {
-    return dedupHash == null ? 0 : fst.nodeCount;
+    return dedupHash == null ? 0 : nodeCount;
   }
 
   private CompiledNode compileNode(UnCompiledNode<T> nodeIn, int tailLength) throws
IOException {
     final long node;
+    long bytesPosStart = bytes.getPosition();
     if (dedupHash != null && (doShareNonSingletonNodes || nodeIn.numArcs <= 1)
&& tailLength <= shareMaxTailLength) {
       if (nodeIn.numArcs == 0) {
-        node = fst.addNode(nodeIn);
+        node = fst.addNode(this, nodeIn);
+        lastFrozenNode = node;
       } else {
-        node = dedupHash.add(nodeIn);
+        node = dedupHash.add(this, nodeIn);
       }
     } else {
-      node = fst.addNode(nodeIn);
+      node = fst.addNode(this, nodeIn);
     }
     assert node != -2;
 
+    long bytesPosEnd = bytes.getPosition();
+    if (bytesPosEnd != bytesPosStart) {
+      // The FST added a new node:
+      assert bytesPosEnd > bytesPosStart;
+      lastFrozenNode = node;
+    }
+
     nodeIn.clear();
 
     final CompiledNode fn = new CompiledNode();
@@ -464,7 +497,7 @@ public class Builder<T> {
     fst.finish(compileNode(root, lastInput.length()).node);
 
     if (doPackFST) {
-      return fst.pack(3, Math.max(10, (int) (fst.getNodeCount()/4)), acceptableOverheadRatio);
+      return fst.pack(this, 3, Math.max(10, (int) (getNodeCount()/4)), acceptableOverheadRatio);
     } else {
       return fst;
     }

Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/util/fst/FST.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/util/fst/FST.java?rev=1688419&r1=1688418&r2=1688419&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/util/fst/FST.java
(original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/util/fst/FST.java
Tue Jun 30 10:07:37 2015
@@ -104,23 +104,20 @@ public final class FST<T> implements Acc
   private static final byte ARCS_AS_FIXED_ARRAY = BIT_ARC_HAS_FINAL_OUTPUT;
 
   /**
-   * @see #shouldExpand(UnCompiledNode)
+   * @see #shouldExpand(Builder, UnCompiledNode)
    */
   static final int FIXED_ARRAY_SHALLOW_DISTANCE = 3; // 0 => only root node.
 
   /**
-   * @see #shouldExpand(UnCompiledNode)
+   * @see #shouldExpand(Builder, UnCompiledNode)
    */
   static final int FIXED_ARRAY_NUM_ARCS_SHALLOW = 5;
 
   /**
-   * @see #shouldExpand(UnCompiledNode)
+   * @see #shouldExpand(Builder, UnCompiledNode)
    */
   static final int FIXED_ARRAY_NUM_ARCS_DEEP = 10;
 
-  // Reused temporarily while building the FST:
-  private int[] reusedBytesPerArc = new int[0];
-
   // Increment version to change it
   private static final String FILE_FORMAT_NAME = "FST";
   private static final int VERSION_START = 0;
@@ -138,7 +135,10 @@ public final class FST<T> implements Acc
    *  Also changed maxBytesPerArc from int to vInt in the array case. */
   private static final int VERSION_VINT_TARGET = 4;
 
-  private static final int VERSION_CURRENT = VERSION_VINT_TARGET;
+  /** Don't store arcWithOutputCount anymore */
+  private static final int VERSION_NO_NODE_ARC_COUNTS = 5;
+
+  private static final int VERSION_CURRENT = VERSION_NO_NODE_ARC_COUNTS;
 
   // Never serialized; just used to represent the virtual
   // final node w/ no arcs:
@@ -157,29 +157,21 @@ public final class FST<T> implements Acc
   // produces this output
   T emptyOutput;
 
+  /** A {@link BytesStore}, used during building, or during reading when
+   *  the FST is very large (more than 1 GB).  If the FST is less than 1
+   *  GB then bytesArray is set instead. */
   final BytesStore bytes;
 
+  /** Used at read time when the FST fits into a single byte[]. */
+  final byte[] bytesArray;
+
   private long startNode = -1;
 
   public final Outputs<T> outputs;
 
-  // Used for the BIT_TARGET_NEXT optimization (whereby
-  // instead of storing the address of the target node for
-  // a given arc, we mark a single bit noting that the next
-  // node in the byte[] is the target node):
-  private long lastFrozenNode;
-
-  private final T NO_OUTPUT;
-
-  public long nodeCount;
-  public long arcCount;
-  public long arcWithOutputCount;
-
   private final boolean packed;
   private PackedInts.Reader nodeRefToAddress;
 
-  private final boolean allowArrayArcs;
-
   private Arc<T> cachedRootArcs[];
 
   /** Represents a single arc. */
@@ -293,16 +285,15 @@ public final class FST<T> implements Acc
 
   // make a new empty FST, for building; Builder invokes
   // this ctor
-  FST(INPUT_TYPE inputType, Outputs<T> outputs, boolean willPackFST, float acceptableOverheadRatio,
boolean allowArrayArcs, int bytesPageBits) {
+  FST(INPUT_TYPE inputType, Outputs<T> outputs, boolean willPackFST, float acceptableOverheadRatio,
int bytesPageBits) {
     this.inputType = inputType;
     this.outputs = outputs;
-    this.allowArrayArcs = allowArrayArcs;
     version = VERSION_CURRENT;
+    bytesArray = null;
     bytes = new BytesStore(bytesPageBits);
     // pad: ensure no node gets address 0 which is reserved to mean
     // the stop state w/ no arcs
     bytes.writeByte((byte) 0);
-    NO_OUTPUT = outputs.getNoOutput();
     if (willPackFST) {
       nodeAddress = new GrowableWriter(15, 8, acceptableOverheadRatio);
       inCounts = new GrowableWriter(1, 8, acceptableOverheadRatio);
@@ -334,7 +325,7 @@ public final class FST<T> implements Acc
 
     // NOTE: only reads most recent format; we don't have
     // back-compat promise for FSTs (they are experimental):
-    version = CodecUtil.checkHeader(in, FILE_FORMAT_NAME, VERSION_PACKED, VERSION_VINT_TARGET);
+    version = CodecUtil.checkHeader(in, FILE_FORMAT_NAME, VERSION_PACKED, VERSION_NO_NODE_ARC_COUNTS);
     packed = in.readByte() == 1;
     if (in.readByte() == 1) {
       // accepts empty string
@@ -380,30 +371,25 @@ public final class FST<T> implements Acc
       nodeRefToAddress = null;
     }
     startNode = in.readVLong();
-    nodeCount = in.readVLong();
-    arcCount = in.readVLong();
-    arcWithOutputCount = in.readVLong();
+    if (version < VERSION_NO_NODE_ARC_COUNTS) {
+      in.readVLong();
+      in.readVLong();
+      in.readVLong();
+    }
 
     long numBytes = in.readVLong();
-    bytes = new BytesStore(in, numBytes, 1<<maxBlockBits);
+    if (numBytes > 1 << maxBlockBits) {
+      // FST is big: we need multiple pages
+      bytes = new BytesStore(in, numBytes, 1<<maxBlockBits);
+      bytesArray = null;
+    } else {
+      // FST fits into a single block: use ByteArrayBytesStoreReader for less overhead
+      bytes = null;
+      bytesArray = new byte[(int) numBytes];
+      in.readBytes(bytesArray, 0, bytesArray.length);
+    }
     
-    NO_OUTPUT = outputs.getNoOutput();
-
     cacheRootArcs();
-
-    // NOTE: bogus because this is only used during
-    // building; we need to break out mutable FST from
-    // immutable
-    allowArrayArcs = false;
-
-    /*
-    if (bytes.length == 665) {
-      Writer w = new OutputStreamWriter(new FileOutputStream("out.dot"), StandardCharsets.UTF_8);
-      Util.toDot(this, w, false, false);
-      w.close();
-      System.out.println("Wrote FST to out.dot");
-    }
-    */
   }
 
   public INPUT_TYPE getInputType() {
@@ -434,7 +420,11 @@ public final class FST<T> implements Acc
   @Override
   public long ramBytesUsed() {
     long size = BASE_RAM_BYTES_USED;
-    size += bytes.ramBytesUsed();
+    if (bytesArray != null) {
+      size += bytesArray.length;
+    } else {
+      size += bytes.ramBytesUsed();
+    }
     if (packed) {
       size += nodeRefToAddress.ramBytesUsed();
     } else if (nodeAddress != null) {
@@ -442,7 +432,6 @@ public final class FST<T> implements Acc
       size += inCounts.ramBytesUsed();
     }
     size += cachedArcsBytesUsed;
-    size += RamUsageEstimator.sizeOf(reusedBytesPerArc);
     return size;
   }
 
@@ -460,10 +449,11 @@ public final class FST<T> implements Acc
 
   @Override
   public String toString() {
-    return getClass().getSimpleName() + "(input=" + inputType + ",output=" + outputs + ",packed="
+ packed + ",nodes=" + nodeCount + ",arcs=" + arcCount + ")";
+    return getClass().getSimpleName() + "(input=" + inputType + ",output=" + outputs + ",packed="
+ packed;
   }
 
   void finish(long newStartNode) throws IOException {
+    assert newStartNode <= bytes.getPosition();
     if (startNode != -1) {
       throw new IllegalStateException("already finished");
     }
@@ -472,7 +462,6 @@ public final class FST<T> implements Acc
     }
     startNode = newStartNode;
     bytes.finish();
-
     cacheRootArcs();
   }
 
@@ -593,12 +582,15 @@ public final class FST<T> implements Acc
       ((PackedInts.Mutable) nodeRefToAddress).save(out);
     }
     out.writeVLong(startNode);
-    out.writeVLong(nodeCount);
-    out.writeVLong(arcCount);
-    out.writeVLong(arcWithOutputCount);
-    long numBytes = bytes.getPosition();
-    out.writeVLong(numBytes);
-    bytes.writeTo(out);
+    if (bytes != null) {
+      long numBytes = bytes.getPosition();
+      out.writeVLong(numBytes);
+      bytes.writeTo(out);
+    } else {
+      assert bytesArray != null;
+      out.writeVLong(bytesArray.length);
+      out.writeBytes(bytesArray, 0, bytesArray.length);
+    }
   }
   
   /**
@@ -655,7 +647,8 @@ public final class FST<T> implements Acc
 
   // serializes new node by appending its bytes to the end
   // of the current byte[]
-  long addNode(Builder.UnCompiledNode<T> nodeIn) throws IOException {
+  long addNode(Builder<T> builder, Builder.UnCompiledNode<T> nodeIn) throws IOException
{
+    T NO_OUTPUT = outputs.getNoOutput();
 
     //System.out.println("FST.addNode pos=" + bytes.getPosition() + " numArcs=" + nodeIn.numArcs);
     if (nodeIn.numArcs == 0) {
@@ -666,22 +659,22 @@ public final class FST<T> implements Acc
       }
     }
 
-    final long startAddress = bytes.getPosition();
+    final long startAddress = builder.bytes.getPosition();
     //System.out.println("  startAddr=" + startAddress);
 
-    final boolean doFixedArray = shouldExpand(nodeIn);
+    final boolean doFixedArray = shouldExpand(builder, nodeIn);
     if (doFixedArray) {
       //System.out.println("  fixedArray");
-      if (reusedBytesPerArc.length < nodeIn.numArcs) {
-        reusedBytesPerArc = new int[ArrayUtil.oversize(nodeIn.numArcs, 1)];
+      if (builder.reusedBytesPerArc.length < nodeIn.numArcs) {
+        builder.reusedBytesPerArc = new int[ArrayUtil.oversize(nodeIn.numArcs, 1)];
       }
     }
 
-    arcCount += nodeIn.numArcs;
+    builder.arcCount += nodeIn.numArcs;
     
     final int lastArc = nodeIn.numArcs-1;
 
-    long lastArcStart = bytes.getPosition();
+    long lastArcStart = builder.bytes.getPosition();
     int maxBytesPerArc = 0;
     for(int arcIdx=0;arcIdx<nodeIn.numArcs;arcIdx++) {
       final Builder.Arc<T> arc = nodeIn.arcs[arcIdx];
@@ -693,7 +686,7 @@ public final class FST<T> implements Acc
         flags += BIT_LAST_ARC;
       }
 
-      if (lastFrozenNode == target.node && !doFixedArray) {
+      if (builder.lastFrozenNode == target.node && !doFixedArray) {
         // TODO: for better perf (but more RAM used) we
         // could avoid this except when arc is "near" the
         // last arc:
@@ -721,36 +714,35 @@ public final class FST<T> implements Acc
         flags += BIT_ARC_HAS_OUTPUT;
       }
 
-      bytes.writeByte((byte) flags);
-      writeLabel(bytes, arc.label);
+      builder.bytes.writeByte((byte) flags);
+      writeLabel(builder.bytes, arc.label);
 
       // System.out.println("  write arc: label=" + (char) arc.label + " flags=" + flags
+ " target=" + target.node + " pos=" + bytes.getPosition() + " output=" + outputs.outputToString(arc.output));
 
       if (arc.output != NO_OUTPUT) {
-        outputs.write(arc.output, bytes);
+        outputs.write(arc.output, builder.bytes);
         //System.out.println("    write output");
-        arcWithOutputCount++;
       }
 
       if (arc.nextFinalOutput != NO_OUTPUT) {
         //System.out.println("    write final output");
-        outputs.writeFinalOutput(arc.nextFinalOutput, bytes);
+        outputs.writeFinalOutput(arc.nextFinalOutput, builder.bytes);
       }
 
       if (targetHasArcs && (flags & BIT_TARGET_NEXT) == 0) {
         assert target.node > 0;
         //System.out.println("    write target");
-        bytes.writeVLong(target.node);
+        builder.bytes.writeVLong(target.node);
       }
 
       // just write the arcs "like normal" on first pass,
       // but record how many bytes each one took, and max
       // byte size:
       if (doFixedArray) {
-        reusedBytesPerArc[arcIdx] = (int) (bytes.getPosition() - lastArcStart);
-        lastArcStart = bytes.getPosition();
-        maxBytesPerArc = Math.max(maxBytesPerArc, reusedBytesPerArc[arcIdx]);
-        //System.out.println("    bytes=" + reusedBytesPerArc[arcIdx]);
+        builder.reusedBytesPerArc[arcIdx] = (int) (builder.bytes.getPosition() - lastArcStart);
+        lastArcStart = builder.bytes.getPosition();
+        maxBytesPerArc = Math.max(maxBytesPerArc, builder.reusedBytesPerArc[arcIdx]);
+        //System.out.println("    bytes=" + builder.reusedBytesPerArc[arcIdx]);
       }
     }
     
@@ -794,53 +786,52 @@ public final class FST<T> implements Acc
       final long fixedArrayStart = startAddress + headerLen;
 
       // expand the arcs in place, backwards
-      long srcPos = bytes.getPosition();
+      long srcPos = builder.bytes.getPosition();
       long destPos = fixedArrayStart + nodeIn.numArcs*maxBytesPerArc;
       assert destPos >= srcPos;
       if (destPos > srcPos) {
-        bytes.skipBytes((int) (destPos - srcPos));
+        builder.bytes.skipBytes((int) (destPos - srcPos));
         for(int arcIdx=nodeIn.numArcs-1;arcIdx>=0;arcIdx--) {
           destPos -= maxBytesPerArc;
-          srcPos -= reusedBytesPerArc[arcIdx];
+          srcPos -= builder.reusedBytesPerArc[arcIdx];
           //System.out.println("  repack arcIdx=" + arcIdx + " srcPos=" + srcPos + " destPos="
+ destPos);
           if (srcPos != destPos) {
-            //System.out.println("  copy len=" + reusedBytesPerArc[arcIdx]);
-            assert destPos > srcPos: "destPos=" + destPos + " srcPos=" + srcPos + " arcIdx="
+ arcIdx + " maxBytesPerArc=" + maxBytesPerArc + " reusedBytesPerArc[arcIdx]=" + reusedBytesPerArc[arcIdx]
+ " nodeIn.numArcs=" + nodeIn.numArcs;
-            bytes.copyBytes(srcPos, destPos, reusedBytesPerArc[arcIdx]);
+            //System.out.println("  copy len=" + builder.reusedBytesPerArc[arcIdx]);
+            assert destPos > srcPos: "destPos=" + destPos + " srcPos=" + srcPos + " arcIdx="
+ arcIdx + " maxBytesPerArc=" + maxBytesPerArc + " reusedBytesPerArc[arcIdx]=" + builder.reusedBytesPerArc[arcIdx]
+ " nodeIn.numArcs=" + nodeIn.numArcs;
+            builder.bytes.copyBytes(srcPos, destPos, builder.reusedBytesPerArc[arcIdx]);
           }
         }
       }
       
       // now write the header
-      bytes.writeBytes(startAddress, header, 0, headerLen);
+      builder.bytes.writeBytes(startAddress, header, 0, headerLen);
     }
 
-    final long thisNodeAddress = bytes.getPosition()-1;
+    final long thisNodeAddress = builder.bytes.getPosition()-1;
 
-    bytes.reverse(startAddress, thisNodeAddress);
+    builder.bytes.reverse(startAddress, thisNodeAddress);
 
     // PackedInts uses int as the index, so we cannot handle
     // > 2.1B nodes when packing:
-    if (nodeAddress != null && nodeCount == Integer.MAX_VALUE) {
+    if (nodeAddress != null && builder.nodeCount == Integer.MAX_VALUE) {
       throw new IllegalStateException("cannot create a packed FST with more than 2.1 billion
nodes");
     }
 
-    nodeCount++;
+    builder.nodeCount++;
     final long node;
     if (nodeAddress != null) {
 
       // Nodes are addressed by 1+ord:
-      if ((int) nodeCount == nodeAddress.size()) {
+      if ((int) builder.nodeCount == nodeAddress.size()) {
         nodeAddress = nodeAddress.resize(ArrayUtil.oversize(nodeAddress.size() + 1, nodeAddress.getBitsPerValue()));
         inCounts = inCounts.resize(ArrayUtil.oversize(inCounts.size() + 1, inCounts.getBitsPerValue()));
       }
-      nodeAddress.set((int) nodeCount, thisNodeAddress);
+      nodeAddress.set((int) builder.nodeCount, thisNodeAddress);
       // System.out.println("  write nodeAddress[" + nodeCount + "] = " + endAddress);
-      node = nodeCount;
+      node = builder.nodeCount;
     } else {
       node = thisNodeAddress;
     }
-    lastFrozenNode = node;
 
     //System.out.println("  ret node=" + node + " address=" + thisNodeAddress + " nodeAddress="
+ nodeAddress);
     return node;
@@ -849,6 +840,7 @@ public final class FST<T> implements Acc
   /** Fills virtual 'start' arc, ie, an empty incoming arc to
    *  the FST's start node */
   public Arc<T> getFirstArc(Arc<T> arc) {
+    T NO_OUTPUT = outputs.getNoOutput();
 
     if (emptyOutput != null) {
       arc.flags = BIT_FINAL_ARC | BIT_LAST_ARC;
@@ -1325,19 +1317,6 @@ public final class FST<T> implements Acc
     }
   }
 
-  public long getNodeCount() {
-    // 1+ in order to count the -1 implicit final node
-    return 1+nodeCount;
-  }
-  
-  public long getArcCount() {
-    return arcCount;
-  }
-
-  public long getArcWithOutputCount() {
-    return arcWithOutputCount;
-  }
-
   /**
    * Nodes will be expanded if their depth (distance from the root node) is
    * &lt;= this value and their number of arcs is &gt;=
@@ -1353,8 +1332,8 @@ public final class FST<T> implements Acc
    * @see #FIXED_ARRAY_NUM_ARCS_DEEP
    * @see Builder.UnCompiledNode#depth
    */
-  private boolean shouldExpand(UnCompiledNode<T> node) {
-    return allowArrayArcs &&
+  private boolean shouldExpand(Builder<T> builder, UnCompiledNode<T> node) {
+    return builder.allowArrayArcs &&
       ((node.depth <= FIXED_ARRAY_SHALLOW_DISTANCE && node.numArcs >= FIXED_ARRAY_NUM_ARCS_SHALLOW)
|| 
        node.numArcs >= FIXED_ARRAY_NUM_ARCS_DEEP);
   }
@@ -1362,13 +1341,19 @@ public final class FST<T> implements Acc
   /** Returns a {@link BytesReader} for this FST, positioned at
    *  position 0. */
   public BytesReader getBytesReader() {
-    BytesReader in;
     if (packed) {
-      in = bytes.getForwardReader();
+      if (bytesArray != null) {
+        return new ForwardBytesReader(bytesArray);
+      } else {
+        return bytes.getForwardReader();
+      }
     } else {
-      in = bytes.getReverseReader();
+      if (bytesArray != null) {
+        return new ReverseBytesReader(bytesArray);
+      } else {
+        return bytes.getReverseReader();
+      }
     }
-    return in;
   }
 
   /** Reads bytes stored in an FST. */
@@ -1497,14 +1482,9 @@ public final class FST<T> implements Acc
     version = VERSION_CURRENT;
     packed = true;
     this.inputType = inputType;
+    bytesArray = null;
     bytes = new BytesStore(bytesPageBits);
     this.outputs = outputs;
-    NO_OUTPUT = outputs.getNoOutput();
-    
-    // NOTE: bogus because this is only used during
-    // building; we need to break out mutable FST from
-    // immutable
-    allowArrayArcs = false;
   }
 
   /** Expert: creates an FST by packing this one.  This
@@ -1519,7 +1499,7 @@ public final class FST<T> implements Acc
    *  However, this is not a strict implementation of the
    *  algorithms described in this paper.
    */
-  FST<T> pack(int minInCountDeref, int maxDerefNodes, float acceptableOverheadRatio)
throws IOException {
+  FST<T> pack(Builder<T> builder, int minInCountDeref, int maxDerefNodes, float
acceptableOverheadRatio) throws IOException {
 
     // NOTE: maxDerefNodes is intentionally int: we cannot
     // support > 2.1B deref nodes
@@ -1539,6 +1519,8 @@ public final class FST<T> implements Acc
       throw new IllegalArgumentException("this FST was not built with willPackFST=true");
     }
 
+    T NO_OUTPUT = outputs.getNoOutput();
+
     Arc<T> arc = new Arc<>();
 
     final BytesReader r = getBytesReader();
@@ -1575,11 +1557,11 @@ public final class FST<T> implements Acc
 
     // +1 because node ords start at 1 (0 is reserved as stop node):
     final GrowableWriter newNodeAddress = new GrowableWriter(
-                       PackedInts.bitsRequired(this.bytes.getPosition()), (int) (1 + nodeCount),
acceptableOverheadRatio);
+                                                             PackedInts.bitsRequired(builder.bytes.getPosition()),
(int) (1 + builder.nodeCount), acceptableOverheadRatio);
 
     // Fill initial coarse guess:
-    for(int node=1;node<=nodeCount;node++) {
-      newNodeAddress.set(node, 1 + this.bytes.getPosition() - nodeAddress.get(node));
+    for(int node=1;node<=builder.nodeCount;node++) {
+      newNodeAddress.set(node, 1 + builder.bytes.getPosition() - nodeAddress.get(node));
     }
 
     int absCount;
@@ -1598,17 +1580,13 @@ public final class FST<T> implements Acc
       // for assert:
       boolean negDelta = false;
 
-      fst = new FST<>(inputType, outputs, bytes.getBlockBits());
+      fst = new FST<>(inputType, outputs, builder.bytes.getBlockBits());
       
       final BytesStore writer = fst.bytes;
 
       // Skip 0 byte since 0 is reserved target:
       writer.writeByte((byte) 0);
 
-      fst.arcWithOutputCount = 0;
-      fst.nodeCount = 0;
-      fst.arcCount = 0;
-
       absCount = deltaCount = topCount = nextCount = 0;
 
       int changedCount = 0;
@@ -1620,8 +1598,7 @@ public final class FST<T> implements Acc
       // Since we re-reverse the bytes, we now write the
       // nodes backwards, so that BIT_TARGET_NEXT is
       // unchanged:
-      for(int node=(int)nodeCount;node>=1;node--) {
-        fst.nodeCount++;
+      for(int node=(int) builder.nodeCount;node>=1;node--) {
         final long address = writer.getPosition();
 
         //System.out.println("  node: " + node + " address=" + address);
@@ -1733,9 +1710,6 @@ public final class FST<T> implements Acc
 
             if (arc.output != NO_OUTPUT) {
               outputs.write(arc.output, writer);
-              if (!retry) {
-                fst.arcWithOutputCount++;
-              }
             }
             if (arc.nextFinalOutput != NO_OUTPUT) {
               outputs.writeFinalOutput(arc.nextFinalOutput, writer);
@@ -1818,8 +1792,6 @@ public final class FST<T> implements Acc
         }
 
         negDelta |= anyNegDelta;
-
-        fst.arcCount += nodeArcCount;
       }
 
       if (!changed) {
@@ -1832,7 +1804,6 @@ public final class FST<T> implements Acc
         // Converged!
         break;
       }
-      //System.out.println("  " + changedCount + " of " + fst.nodeCount + " changed; retry");
     }
 
     long maxAddress = 0;
@@ -1854,10 +1825,6 @@ public final class FST<T> implements Acc
       fst.setEmptyOutput(emptyOutput);
     }
 
-    assert fst.nodeCount == nodeCount: "fst.nodeCount=" + fst.nodeCount + " nodeCount=" +
nodeCount;
-    assert fst.arcCount == arcCount;
-    assert fst.arcWithOutputCount == arcWithOutputCount: "fst.arcWithOutputCount=" + fst.arcWithOutputCount
+ " arcWithOutputCount=" + arcWithOutputCount;
-
     fst.bytes.finish();
     fst.cacheRootArcs();
 

Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/util/fst/NodeHash.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/util/fst/NodeHash.java?rev=1688419&r1=1688418&r2=1688419&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/util/fst/NodeHash.java
(original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/util/fst/NodeHash.java
Tue Jun 30 10:07:37 2015
@@ -114,7 +114,7 @@ final class NodeHash<T> {
     return h & Long.MAX_VALUE;
   }
 
-  public long add(Builder.UnCompiledNode<T> nodeIn) throws IOException {
+  public long add(Builder<T> builder, Builder.UnCompiledNode<T> nodeIn) throws
IOException {
     //System.out.println("hash: add count=" + count + " vs " + table.size() + " mask=" +
mask);
     final long h = hash(nodeIn);
     long pos = h & mask;
@@ -123,7 +123,7 @@ final class NodeHash<T> {
       final long v = table.get(pos);
       if (v == 0) {
         // freeze & add
-        final long node = fst.addNode(nodeIn);
+        final long node = fst.addNode(builder, nodeIn);
         //System.out.println("  now freeze node=" + node);
         assert hash(node) == h : "frozenHash=" + hash(node) + " vs h=" + h;
         count++;

Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/util/fst/Test2BFST.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/util/fst/Test2BFST.java?rev=1688419&r1=1688418&r2=1688419&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/util/fst/Test2BFST.java
(original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/util/fst/Test2BFST.java
Tue Jun 30 10:07:37 2015
@@ -69,9 +69,9 @@ public class Test2BFST extends LuceneTes
           b.add(input2, NO_OUTPUT);
           count++;
           if (count % 100000 == 0) {
-            System.out.println(count + ": " + b.fstRamBytesUsed() + " bytes; " + b.getTotStateCount()
+ " nodes");
+            System.out.println(count + ": " + b.fstRamBytesUsed() + " bytes; " + b.getNodeCount()
+ " nodes");
           }
-          if (b.getTotStateCount() > Integer.MAX_VALUE + 100L * 1024 * 1024) {
+          if (b.getNodeCount() > Integer.MAX_VALUE + 100L * 1024 * 1024) {
             break;
           }
           nextInput(r, ints2);
@@ -80,7 +80,7 @@ public class Test2BFST extends LuceneTes
         FST<Object> fst = b.finish();
 
         for(int verify=0;verify<2;verify++) {
-          System.out.println("\nTEST: now verify [fst size=" + fst.ramBytesUsed() + "; nodeCount="
+ fst.getNodeCount() + "; arcCount=" + fst.getArcCount() + "]");
+          System.out.println("\nTEST: now verify [fst size=" + fst.ramBytesUsed() + "; nodeCount="
+ b.getNodeCount() + "; arcCount=" + b.getArcCount() + "]");
 
           Arrays.fill(ints2, 0);
           r = new Random(seed);
@@ -161,7 +161,7 @@ public class Test2BFST extends LuceneTes
         FST<BytesRef> fst = b.finish();
         for(int verify=0;verify<2;verify++) {
 
-          System.out.println("\nTEST: now verify [fst size=" + fst.ramBytesUsed() + "; nodeCount="
+ fst.getNodeCount() + "; arcCount=" + fst.getArcCount() + "]");
+          System.out.println("\nTEST: now verify [fst size=" + fst.ramBytesUsed() + "; nodeCount="
+ b.getNodeCount() + "; arcCount=" + b.getArcCount() + "]");
 
           r = new Random(seed);
           Arrays.fill(ints, 0);
@@ -239,7 +239,7 @@ public class Test2BFST extends LuceneTes
 
         for(int verify=0;verify<2;verify++) {
 
-          System.out.println("\nTEST: now verify [fst size=" + fst.ramBytesUsed() + "; nodeCount="
+ fst.getNodeCount() + "; arcCount=" + fst.getArcCount() + "]");
+          System.out.println("\nTEST: now verify [fst size=" + fst.ramBytesUsed() + "; nodeCount="
+ b.getNodeCount() + "; arcCount=" + b.getArcCount() + "]");
 
           Arrays.fill(ints, 0);
 

Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java?rev=1688419&r1=1688418&r2=1688419&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java
(original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java
Tue Jun 30 10:07:37 2015
@@ -133,10 +133,11 @@ public class TestFSTs extends LuceneTest
         for(IntsRef term : terms2) {
           pairs.add(new FSTTester.InputOutput<>(term, NO_OUTPUT));
         }
-        FST<Object> fst = new FSTTester<>(random(), dir, inputMode, pairs, outputs,
false).doTest(0, 0, false);
+        FSTTester<Object> tester = new FSTTester<>(random(), dir, inputMode,
pairs, outputs, false);
+        FST<Object> fst = tester.doTest(0, 0, false);
         assertNotNull(fst);
-        assertEquals(22, fst.getNodeCount());
-        assertEquals(27, fst.getArcCount());
+        assertEquals(22, tester.nodeCount);
+        assertEquals(27, tester.arcCount);
       }
 
       // FST ord pos int
@@ -146,10 +147,11 @@ public class TestFSTs extends LuceneTest
         for(int idx=0;idx<terms2.length;idx++) {
           pairs.add(new FSTTester.InputOutput<>(terms2[idx], (long) idx));
         }
-        final FST<Long> fst = new FSTTester<>(random(), dir, inputMode, pairs,
outputs, true).doTest(0, 0, false);
+        FSTTester<Long> tester = new FSTTester<>(random(), dir, inputMode, pairs,
outputs, true);
+        final FST<Long> fst = tester.doTest(0, 0, false);
         assertNotNull(fst);
-        assertEquals(22, fst.getNodeCount());
-        assertEquals(27, fst.getArcCount());
+        assertEquals(22, tester.nodeCount);
+        assertEquals(27, tester.arcCount);
       }
 
       // FST byte sequence ord
@@ -161,10 +163,11 @@ public class TestFSTs extends LuceneTest
           final BytesRef output = random().nextInt(30) == 17 ? NO_OUTPUT : new BytesRef(Integer.toString(idx));
           pairs.add(new FSTTester.InputOutput<>(terms2[idx], output));
         }
-        final FST<BytesRef> fst = new FSTTester<>(random(), dir, inputMode, pairs,
outputs, false).doTest(0, 0, false);
+        FSTTester<BytesRef> tester = new FSTTester<>(random(), dir, inputMode,
pairs, outputs, false);
+        final FST<BytesRef> fst = tester.doTest(0, 0, false);
         assertNotNull(fst);
-        assertEquals(24, fst.getNodeCount());
-        assertEquals(30, fst.getArcCount());
+        assertEquals(24, tester.nodeCount);
+        assertEquals(30, tester.arcCount);
       }
     }
   }
@@ -383,7 +386,7 @@ public class TestFSTs extends LuceneTest
       }
       FST<Long> fst = builder.finish();
       if (VERBOSE) {
-        System.out.println("FST: " + docCount + " docs; " + ord + " terms; " + fst.getNodeCount()
+ " nodes; " + fst.getArcCount() + " arcs;" + " " + fst.ramBytesUsed() + " bytes");
+        System.out.println("FST: " + docCount + " docs; " + ord + " terms; " + builder.getNodeCount()
+ " nodes; " + builder.getArcCount() + " arcs;" + " " + fst.ramBytesUsed() + " bytes");
       }
 
       if (ord > 0) {
@@ -520,8 +523,8 @@ public class TestFSTs extends LuceneTest
           return;
         }
 
-        System.out.println(ord + " terms; " + fst.getNodeCount() + " nodes; " + fst.getArcCount()
+ " arcs; " + fst.getArcWithOutputCount() + " arcs w/ output; tot size " + fst.ramBytesUsed());
-        if (fst.getNodeCount() < 100) {
+        System.out.println(ord + " terms; " + builder.getNodeCount() + " nodes; " + builder.getArcCount()
+ " arcs; tot size " + fst.ramBytesUsed());
+        if (builder.getNodeCount() < 100) {
           Writer w = Files.newBufferedWriter(Paths.get("out.dot"), StandardCharsets.UTF_8);
           Util.toDot(fst, w, false, false);
           w.close();
@@ -1155,7 +1158,8 @@ public class TestFSTs extends LuceneTest
     final Long nothing = outputs.getNoOutput();
     final Builder<Long> b = new Builder<>(FST.INPUT_TYPE.BYTE1, outputs);
 
-    final FST<Long> fst = new FST<>(FST.INPUT_TYPE.BYTE1, outputs, false, PackedInts.COMPACT,
true, 15);
+    //final FST<Long> fst = new FST<>(FST.INPUT_TYPE.BYTE1, outputs, false, PackedInts.COMPACT,
15);
+    final FST<Long> fst = b.fst;
 
     final Builder.UnCompiledNode<Long> rootNode = new Builder.UnCompiledNode<>(b,
0);
 
@@ -1165,7 +1169,7 @@ public class TestFSTs extends LuceneTest
       node.isFinal = true;
       rootNode.addArc('a', node);
       final Builder.CompiledNode frozen = new Builder.CompiledNode();
-      frozen.node = fst.addNode(node);
+      frozen.node = fst.addNode(b, node);
       rootNode.arcs[0].nextFinalOutput = 17L;
       rootNode.arcs[0].isFinal = true;
       rootNode.arcs[0].output = nothing;
@@ -1177,13 +1181,13 @@ public class TestFSTs extends LuceneTest
       final Builder.UnCompiledNode<Long> node = new Builder.UnCompiledNode<>(b,
0);
       rootNode.addArc('b', node);
       final Builder.CompiledNode frozen = new Builder.CompiledNode();
-      frozen.node = fst.addNode(node);
+      frozen.node = fst.addNode(b, node);
       rootNode.arcs[1].nextFinalOutput = nothing;
       rootNode.arcs[1].output = 42L;
       rootNode.arcs[1].target = frozen;
     }
 
-    fst.finish(fst.addNode(rootNode));
+    fst.finish(fst.addNode(b, rootNode));
 
     StringWriter w = new StringWriter();
     //Writer w = new OutputStreamWriter(new FileOutputStream("/x/tmp3/out.dot"));

Modified: lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/util/fst/FSTTester.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/util/fst/FSTTester.java?rev=1688419&r1=1688418&r2=1688419&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/util/fst/FSTTester.java
(original)
+++ lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/util/fst/FSTTester.java
Tue Jun 30 10:07:37 2015
@@ -58,6 +58,8 @@ public class FSTTester<T> {
   final Outputs<T> outputs;
   final Directory dir;
   final boolean doReverseLookup;
+  long nodeCount;
+  long arcCount;
 
   public FSTTester(Random random, Directory dir, int inputMode, List<InputOutput<T>>
pairs, Outputs<T> outputs, boolean doReverseLookup) {
     this.random = random;
@@ -331,7 +333,7 @@ public class FSTTester<T> {
       if (fst == null) {
         System.out.println("  fst has 0 nodes (fully pruned)");
       } else {
-        System.out.println("  fst has " + fst.getNodeCount() + " nodes and " + fst.getArcCount()
+ " arcs");
+        System.out.println("  fst has " + builder.getNodeCount() + " nodes and " + builder.getArcCount()
+ " arcs");
       }
     }
 
@@ -341,6 +343,9 @@ public class FSTTester<T> {
       verifyPruned(inputMode, fst, prune1, prune2);
     }
 
+    nodeCount = builder.getNodeCount();
+    arcCount = builder.getArcCount();
+
     return fst;
   }
 



Mime
View raw message