lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From romseyg...@apache.org
Subject [2/2] lucene-solr:branch_7x: LUCENE-8471: Add IndexWriter.getFlushingBytes() method
Date Thu, 30 Aug 2018 09:28:44 GMT
LUCENE-8471: Add IndexWriter.getFlushingBytes() method


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/06d6e63f
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/06d6e63f
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/06d6e63f

Branch: refs/heads/branch_7x
Commit: 06d6e63fdac888578545834116be6842fc12f65e
Parents: bf7d107
Author: Alan Woodward <romseygeek@apache.org>
Authored: Thu Aug 30 09:56:09 2018 +0100
Committer: Alan Woodward <romseygeek@apache.org>
Committed: Thu Aug 30 10:12:00 2018 +0100

----------------------------------------------------------------------
 lucene/CHANGES.txt                                    |  3 +++
 .../java/org/apache/lucene/index/DocumentsWriter.java |  7 +++++++
 .../lucene/index/DocumentsWriterFlushControl.java     |  8 ++++----
 .../src/java/org/apache/lucene/index/IndexWriter.java |  8 ++++++++
 .../lucene/index/TestFlushByRamOrCountsPolicy.java    | 14 +++++++-------
 5 files changed, 29 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/06d6e63f/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index bc7aeda..541a754 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -55,6 +55,9 @@ API Changes:
   word-by-word.  The UH's public internal APIs changed a bit in the process.
   (David Smiley)
 
+* LUCENE-8471: IndexWriter.getFlushingBytes() returns how many bytes are currently
+  being flushed to disk. (Alan Woodward)
+
 Bug Fixes:
 
 * LUCENE-8445: Tighten condition when two planes are identical to prevent constructing

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/06d6e63f/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java
index cbbf22e..136bd81 100644
--- a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java
@@ -762,4 +762,11 @@ final class DocumentsWriter implements Closeable, Accountable {
   public long ramBytesUsed() {
     return flushControl.ramBytesUsed();
   }
+
+  /**
+   * Returns the number of bytes currently being flushed
+   */
+  public long getFlushingBytes() {
+    return flushControl.getFlushingBytes();
+  }
 }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/06d6e63f/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java
b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java
index ad5b7e4..5b6f8af 100644
--- a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java
+++ b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java
@@ -47,7 +47,7 @@ final class DocumentsWriterFlushControl implements Accountable {
 
   private final long hardMaxBytesPerDWPT;
   private long activeBytes = 0;
-  private long flushBytes = 0;
+  private volatile long flushBytes = 0;
   private volatile int numPending = 0;
   private int numDocsSinceStalled = 0; // only with assert
   final AtomicBoolean flushDeletes = new AtomicBoolean(false);
@@ -86,7 +86,7 @@ final class DocumentsWriterFlushControl implements Accountable {
     return activeBytes;
   }
 
-  public synchronized long flushBytes() {
+  public long getFlushingBytes() {
     return flushBytes;
   }
 
@@ -257,11 +257,11 @@ final class DocumentsWriterFlushControl implements Accountable {
       if (stall != stallControl.anyStalledThreads()) {
         if (stall) {
           infoStream.message("DW", String.format(Locale.ROOT, "now stalling flushes: netBytes:
%.1f MB flushBytes: %.1f MB fullFlush: %b",
-                                                 netBytes()/1024./1024., flushBytes()/1024./1024.,
fullFlush));
+                                                 netBytes()/1024./1024., getFlushingBytes()/1024./1024.,
fullFlush));
           stallStartNS = System.nanoTime();
         } else {
           infoStream.message("DW", String.format(Locale.ROOT, "done stalling flushes for
%.1f msec: netBytes: %.1f MB flushBytes: %.1f MB fullFlush: %b",
-                                                 (System.nanoTime()-stallStartNS)/1000000.,
netBytes()/1024./1024., flushBytes()/1024./1024., fullFlush));
+                                                 (System.nanoTime()-stallStartNS)/1000000.,
netBytes()/1024./1024., getFlushingBytes()/1024./1024., fullFlush));
         }
       }
     }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/06d6e63f/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
index fbc9f37..0ce5882 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
@@ -570,6 +570,14 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable,
     return docWriter.ramBytesUsed();
   }
 
+  /**
+   * Returns the number of bytes currently being flushed
+   */
+  public final long getFlushingBytes() {
+    ensureOpen();
+    return docWriter.getFlushingBytes();
+  }
+
   final long getReaderPoolRamBytesUsed() {
     return readerPool.ramBytesUsed();
   }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/06d6e63f/lucene/core/src/test/org/apache/lucene/index/TestFlushByRamOrCountsPolicy.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestFlushByRamOrCountsPolicy.java
b/lucene/core/src/test/org/apache/lucene/index/TestFlushByRamOrCountsPolicy.java
index ca5aba8..fa6cfa9 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestFlushByRamOrCountsPolicy.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestFlushByRamOrCountsPolicy.java
@@ -81,7 +81,7 @@ public class TestFlushByRamOrCountsPolicy extends LuceneTestCase {
     DocumentsWriter docsWriter = writer.getDocsWriter();
     assertNotNull(docsWriter);
     DocumentsWriterFlushControl flushControl = docsWriter.flushControl;
-    assertEquals(" bytes must be 0 after init", 0, flushControl.flushBytes());
+    assertEquals(" bytes must be 0 after init", 0, writer.getFlushingBytes());
 
     IndexThread[] threads = new IndexThread[numThreads];
     for (int x = 0; x < threads.length; x++) {
@@ -95,7 +95,7 @@ public class TestFlushByRamOrCountsPolicy extends LuceneTestCase {
     }
     final long maxRAMBytes = (long) (iwc.getRAMBufferSizeMB() * 1024. * 1024.);
     assertEquals(" all flushes must be due numThreads=" + numThreads, 0,
-        flushControl.flushBytes());
+        writer.getFlushingBytes());
     assertEquals(numDocumentsToIndex, writer.numDocs());
     assertEquals(numDocumentsToIndex, writer.maxDoc());
     assertTrue("peak bytes without flush exceeded watermark",
@@ -136,7 +136,7 @@ public class TestFlushByRamOrCountsPolicy extends LuceneTestCase {
       DocumentsWriter docsWriter = writer.getDocsWriter();
       assertNotNull(docsWriter);
       DocumentsWriterFlushControl flushControl = docsWriter.flushControl;
-      assertEquals(" bytes must be 0 after init", 0, flushControl.flushBytes());
+      assertEquals(" bytes must be 0 after init", 0, writer.getFlushingBytes());
 
       IndexThread[] threads = new IndexThread[numThreads[i]];
       for (int x = 0; x < threads.length; x++) {
@@ -150,7 +150,7 @@ public class TestFlushByRamOrCountsPolicy extends LuceneTestCase {
       }
 
       assertEquals(" all flushes must be due numThreads=" + numThreads[i], 0,
-          flushControl.flushBytes());
+          writer.getFlushingBytes());
       assertEquals(numDocumentsToIndex, writer.numDocs());
       assertEquals(numDocumentsToIndex, writer.maxDoc());
       assertTrue("peak bytes without flush exceeded watermark",
@@ -182,7 +182,7 @@ public class TestFlushByRamOrCountsPolicy extends LuceneTestCase {
     assertNotNull(docsWriter);
     DocumentsWriterFlushControl flushControl = docsWriter.flushControl;
 
-    assertEquals(" bytes must be 0 after init", 0, flushControl.flushBytes());
+    assertEquals(" bytes must be 0 after init", 0, writer.getFlushingBytes());
 
     IndexThread[] threads = new IndexThread[numThreads];
     for (int x = 0; x < threads.length; x++) {
@@ -194,7 +194,7 @@ public class TestFlushByRamOrCountsPolicy extends LuceneTestCase {
     for (int x = 0; x < threads.length; x++) {
       threads[x].join();
     }
-    assertEquals(" all flushes must be due", 0, flushControl.flushBytes());
+    assertEquals(" all flushes must be due", 0, writer.getFlushingBytes());
     assertEquals(numDocumentsToIndex, writer.numDocs());
     assertEquals(numDocumentsToIndex, writer.maxDoc());
     if (flushPolicy.flushOnRAM() && !flushPolicy.flushOnDocCount()) {
@@ -255,7 +255,7 @@ public class TestFlushByRamOrCountsPolicy extends LuceneTestCase {
       DocumentsWriter docsWriter = writer.getDocsWriter();
       assertNotNull(docsWriter);
       DocumentsWriterFlushControl flushControl = docsWriter.flushControl;
-      assertEquals(" all flushes must be due", 0, flushControl.flushBytes());
+      assertEquals(" all flushes must be due", 0, writer.getFlushingBytes());
       assertEquals(numDocumentsToIndex, writer.numDocs());
       assertEquals(numDocumentsToIndex, writer.maxDoc());
       if (numThreads[i] == 1) {


Mime
View raw message