lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a.@apache.org
Subject [23/23] lucene-solr:feature/metrics: Merge branch 'master' into feature/metrics
Date Mon, 19 Dec 2016 12:52:07 GMT
Merge branch 'master' into feature/metrics


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/67dc7604
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/67dc7604
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/67dc7604

Branch: refs/heads/feature/metrics
Commit: 67dc7604ef7340cf94ad286f3693977aec12f26f
Parents: f56da1d 321c6f0
Author: Andrzej Bialecki <ab@apache.org>
Authored: Mon Dec 19 13:51:25 2016 +0100
Committer: Andrzej Bialecki <ab@apache.org>
Committed: Mon Dec 19 13:51:25 2016 +0100

----------------------------------------------------------------------
 lucene/CHANGES.txt                              |  17 +
 .../lucene70/Lucene70DocValuesConsumer.java     | 163 +++++--
 .../lucene70/Lucene70DocValuesFormat.java       |   5 +-
 .../lucene70/Lucene70DocValuesProducer.java     | 220 +++++++--
 .../org/apache/lucene/index/LeafReader.java     |   2 +-
 .../apache/lucene/index/PrefixCodedTerms.java   |   4 +-
 .../org/apache/lucene/index/SegmentInfos.java   |   7 +-
 .../java/org/apache/lucene/util/LongValues.java |   9 +
 .../apache/lucene/util/packed/DirectWriter.java |   8 +-
 .../lucene70/TestLucene70DocValuesFormat.java   | 152 +++++++
 .../org/apache/lucene/facet/FacetQuery.java     |  52 +++
 .../apache/lucene/facet/MultiFacetQuery.java    |  61 +++
 .../org/apache/lucene/facet/TestFacetQuery.java | 103 +++++
 .../apache/lucene/search/DocValuesStats.java    | 409 +++++++++++++++++
 .../lucene/search/DocValuesStatsCollector.java  |  64 +++
 .../search/TestDocValuesStatsCollector.java     | 448 +++++++++++++++++++
 .../apache/lucene/document/RangeFieldQuery.java |   1 +
 .../lucene/search/DocValuesNumbersQuery.java    |  26 +-
 .../lucene/search/DocValuesTermsQuery.java      |  49 +-
 .../org/apache/lucene/search/LongHashSet.java   | 156 +++++++
 .../search/BaseRangeFieldQueryTestCase.java     |   2 +-
 .../apache/lucene/search/LongHashSetTests.java  | 100 +++++
 .../lucene/search/TestDocValuesTermsQuery.java  |   1 +
 .../org/apache/lucene/search/QueryUtils.java    |   4 -
 solr/CHANGES.txt                                |  14 +
 .../solr/collection1/conf/solrconfig.xml        |   5 -
 solr/contrib/ltr/example/solrconfig.xml         |  11 -
 .../solr/solrcloud/conf/solrconfig.xml          |  11 -
 .../OverseerAutoReplicaFailoverThread.java      |   9 +-
 .../java/org/apache/solr/core/SolrConfig.java   |   2 +-
 .../src/java/org/apache/solr/core/SolrCore.java |  92 ++--
 .../apache/solr/search/SolrFieldCacheMBean.java |   6 +-
 .../apache/solr/servlet/SolrDispatchFilter.java |  15 +
 .../apache/solr/servlet/SolrRequestParsers.java |  31 +-
 .../solr/uninverting/UninvertingReader.java     |  25 +-
 .../solr/util/SolrFileCleaningTracker.java      | 147 ++++++
 .../solr/collection1/conf/solrconfig_perf.xml   |   1 -
 .../cloud/SegmentTerminateEarlyTestState.java   |  12 +-
 .../solr/cloud/TestMiniSolrCloudCluster.java    |  50 ---
 .../apache/solr/cloud/TestSegmentSorting.java   | 133 ++++++
 .../org/apache/solr/search/TestRealTimeGet.java |   2 +-
 .../solr/search/TestSolrFieldCacheMBean.java    |   3 +-
 .../apache/solr/search/TestStressLucene.java    |   2 +-
 .../apache/solr/search/TestStressRecovery.java  |   2 +-
 .../apache/solr/search/TestStressReorder.java   |   5 +-
 .../solr/search/TestStressUserVersions.java     |   4 +-
 .../apache/solr/search/TestStressVersions.java  |   2 +-
 .../example-DIH/solr/db/conf/solrconfig.xml     |  11 -
 .../example-DIH/solr/mail/conf/solrconfig.xml   |  11 -
 .../example-DIH/solr/rss/conf/solrconfig.xml    |  11 -
 .../example-DIH/solr/solr/conf/solrconfig.xml   |  11 -
 .../example-DIH/solr/tika/conf/solrconfig.xml   |  11 -
 solr/example/files/conf/solrconfig.xml          |  11 -
 .../basic_configs/conf/solrconfig.xml           |  11 -
 .../conf/solrconfig.xml                         |  11 -
 .../conf/solrconfig.xml                         |  11 -
 .../solr/client/solrj/LargeVolumeTestBase.java  |   3 +-
 .../solr/client/solrj/SolrExampleTestBase.java  |   1 -
 .../java/org/apache/solr/SolrJettyTestBase.java |   2 -
 .../java/org/apache/solr/SolrTestCaseJ4.java    |   2 +-
 60 files changed, 2371 insertions(+), 383 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/67dc7604/solr/CHANGES.txt
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/67dc7604/solr/core/src/java/org/apache/solr/core/SolrCore.java
----------------------------------------------------------------------
diff --cc solr/core/src/java/org/apache/solr/core/SolrCore.java
index cad6423,b3facef..4a63686
--- a/solr/core/src/java/org/apache/solr/core/SolrCore.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java
@@@ -206,12 -200,6 +206,12 @@@ public final class SolrCore implements 
    private final ReentrantLock ruleExpiryLock;
    private final ReentrantLock snapshotDelLock; // A lock instance to guard against concurrent
deletions.
  
 +  private final Timer newSearcherTimer;
 +  private final Timer newSearcherWarmupTimer;
 +  private final Counter newSearcherCounter;
-   private final Counter newSearcherMaxErrorsCounter;
++  private final Counter newSearcherMaxReachedCounter;
 +  private final Counter newSearcherOtherErrorsCounter;
 +
    public Date getStartTimeStamp() { return startTime; }
  
    private final Map<Object, IndexFingerprint> perSegmentFingerprintCache = new MapMaker().weakKeys().makeMap();
@@@ -863,16 -838,6 +863,16 @@@
  
      checkVersionFieldExistsInSchema(schema, coreDescriptor);
  
 +    // Initialize the metrics manager
 +    this.metricManager = initMetricManager(config);
 +
 +    // initialize searcher-related metrics
 +    newSearcherCounter = SolrMetricManager.counter(metricManager.getRegistryName(), "newSearcher");
 +    newSearcherTimer = SolrMetricManager.timer(metricManager.getRegistryName(), "newSearcherTime");
 +    newSearcherWarmupTimer = SolrMetricManager.timer(metricManager.getRegistryName(), "newSearcherWarmup");
-     newSearcherMaxErrorsCounter = SolrMetricManager.counter(metricManager.getRegistryName(),
"newSearcherMaxErrors");
++    newSearcherMaxReachedCounter = SolrMetricManager.counter(metricManager.getRegistryName(),
"newSearcherMaxReached");
 +    newSearcherOtherErrorsCounter = SolrMetricManager.counter(metricManager.getRegistryName(),
"newSearcherErrors");
 +
      // Initialize JMX
      this.infoRegistry = initInfoRegistry(name, config);
      infoRegistry.put("fieldCache", new SolrFieldCacheMBean());
@@@ -1943,53 -1886,57 +1943,59 @@@
      // if it isn't necessary.
  
      synchronized (searcherLock) {
-       // see if we can return the current searcher
-       if (_searcher!=null && !forceNew) {
-         if (returnSearcher) {
-           _searcher.incref();
-           return _searcher;
-         } else {
-           return null;
+       for(;;) { // this loop is so w can retry in the event that we exceed maxWarmingSearchers
+         // see if we can return the current searcher
+         if (_searcher != null && !forceNew) {
+           if (returnSearcher) {
+             _searcher.incref();
+             return _searcher;
+           } else {
+             return null;
+           }
          }
-       }
  
-       // check to see if we can wait for someone else's searcher to be set
-       if (onDeckSearchers>0 && !forceNew && _searcher==null) {
-         try {
-           searcherLock.wait();
-         } catch (InterruptedException e) {
-           log.info(SolrException.toStr(e));
+         // check to see if we can wait for someone else's searcher to be set
+         if (onDeckSearchers > 0 && !forceNew && _searcher == null) {
+           try {
+             searcherLock.wait();
+           } catch (InterruptedException e) {
+             log.info(SolrException.toStr(e));
+           }
          }
-       }
  
-       // check again: see if we can return right now
-       if (_searcher!=null && !forceNew) {
-         if (returnSearcher) {
-           _searcher.incref();
-           return _searcher;
-         } else {
-           return null;
+         // check again: see if we can return right now
+         if (_searcher != null && !forceNew) {
+           if (returnSearcher) {
+             _searcher.incref();
+             return _searcher;
+           } else {
+             return null;
+           }
+         }
+ 
+         // At this point, we know we need to open a new searcher...
+         // first: increment count to signal other threads that we are
+         //        opening a new searcher.
+         onDeckSearchers++;
++        newSearcherCounter.inc();
+         if (onDeckSearchers < 1) {
+           // should never happen... just a sanity check
+           log.error(logid + "ERROR!!! onDeckSearchers is " + onDeckSearchers);
+           onDeckSearchers = 1;  // reset
+         } else if (onDeckSearchers > maxWarmingSearchers) {
+           onDeckSearchers--;
++          newSearcherMaxReachedCounter.inc();
+           try {
+             searcherLock.wait();
+           } catch (InterruptedException e) {
+             log.info(SolrException.toStr(e));
+           }
+           continue;  // go back to the top of the loop and retry
+         } else if (onDeckSearchers > 1) {
+           log.warn(logid + "PERFORMANCE WARNING: Overlapping onDeckSearchers=" + onDeckSearchers);
          }
-       }
  
-       // At this point, we know we need to open a new searcher...
-       // first: increment count to signal other threads that we are
-       //        opening a new searcher.
-       onDeckSearchers++;
-       newSearcherCounter.inc();
-       if (onDeckSearchers < 1) {
-         // should never happen... just a sanity check
-         log.error(logid+"ERROR!!! onDeckSearchers is " + onDeckSearchers);
-         onDeckSearchers=1;  // reset
-       } else if (onDeckSearchers > maxWarmingSearchers) {
-         onDeckSearchers--;
-         String msg="Error opening new searcher. exceeded limit of maxWarmingSearchers="+maxWarmingSearchers
+ ", try again later.";
-         log.warn(logid+""+ msg);
-         // HTTP 503==service unavailable, or 409==Conflict
-         newSearcherMaxErrorsCounter.inc();
-         throw new SolrException(SolrException.ErrorCode.SERVICE_UNAVAILABLE,msg);
-       } else if (onDeckSearchers > 1) {
-         log.warn(logid+"PERFORMANCE WARNING: Overlapping onDeckSearchers=" + onDeckSearchers);
+         break; // I can now exit the loop and proceed to open a searcher
        }
      }
  

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/67dc7604/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
----------------------------------------------------------------------
diff --cc solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
index cfd90cb,e8c4657..2852041
--- a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
+++ b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
@@@ -47,12 -45,7 +47,13 @@@ import java.util.concurrent.atomic.Atom
  import java.util.regex.Matcher;
  import java.util.regex.Pattern;
  
 +import com.codahale.metrics.jvm.BufferPoolMetricSet;
 +import com.codahale.metrics.jvm.ClassLoadingGaugeSet;
 +import com.codahale.metrics.jvm.FileDescriptorRatioGauge;
 +import com.codahale.metrics.jvm.GarbageCollectorMetricSet;
 +import com.codahale.metrics.jvm.MemoryUsageGaugeSet;
 +import com.codahale.metrics.jvm.ThreadStatesGaugeSet;
+ import org.apache.commons.io.FileCleaningTracker;
  import org.apache.commons.io.input.CloseShieldInputStream;
  import org.apache.commons.io.output.CloseShieldOutputStream;
  import org.apache.commons.lang.StringUtils;


Mime
View raw message