lucene-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Alan Woodward <alan.woodw...@romseysoftware.co.uk>
Subject Re: svn commit: r1403555 - in /lucene/dev/trunk/solr: ./ core/ core/src/java/org/apache/solr/handler/ core/src/test/org/apache/solr/core/ test-framework/src/java/org/apache/solr/
Date Tue, 30 Oct 2012 08:39:52 GMT
Aargh, sorry about that, forgot to run ant precommit.

Apologies for the noise, everyone.

On 29 Oct 2012, at 23:02, Robert Muir wrote:

> the new ivy jar will want .sha1's and licenses files. (builds will fail)
> 
> you can generate the .sha1s with ant jar-checksums. but the
> license/notice you have to put in licenses/ manually.
> 
> On Mon, Oct 29, 2012 at 6:13 PM,  <romseygeek@apache.org> wrote:
>> Author: romseygeek
>> Date: Mon Oct 29 22:13:03 2012
>> New Revision: 1403555
>> 
>> URL: http://svn.apache.org/viewvc?rev=1403555&view=rev
>> Log:
>> SOLR-1972: Add extra query stats to RequestHandler
>> 
>> Modified:
>>    lucene/dev/trunk/solr/CHANGES.txt
>>    lucene/dev/trunk/solr/core/ivy.xml
>>    lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java
>>    lucene/dev/trunk/solr/core/src/test/org/apache/solr/core/RequestHandlersTest.java
>>    lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/SolrIgnoredThreadsFilter.java
>> 
>> Modified: lucene/dev/trunk/solr/CHANGES.txt
>> URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=1403555&r1=1403554&r2=1403555&view=diff
>> ==============================================================================
>> --- lucene/dev/trunk/solr/CHANGES.txt (original)
>> +++ lucene/dev/trunk/solr/CHANGES.txt Mon Oct 29 22:13:03 2012
>> @@ -55,6 +55,10 @@ New Features
>> * SOLR-3911: Make Directory and DirectoryFactory first class so that the majority
>>   of Solr's features work with any custom implementations. (Mark Miller)
>> 
>> +* SOLR-1972: Add extra statistics to RequestHandlers - 5 & 15-minute reqs/sec
>> +  rolling averages; median, 75th, 95th, 99th, 99.9th percentile request times
>> +  (Alan Woodward)
>> +
>> Optimizations
>> ----------------------
>> 
>> 
>> Modified: lucene/dev/trunk/solr/core/ivy.xml
>> URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/ivy.xml?rev=1403555&r1=1403554&r2=1403555&view=diff
>> ==============================================================================
>> --- lucene/dev/trunk/solr/core/ivy.xml (original)
>> +++ lucene/dev/trunk/solr/core/ivy.xml Mon Oct 29 22:13:03 2012
>> @@ -28,6 +28,7 @@
>>       <dependency org="org.easymock" name="easymock" rev="2.2" transitive="false"/>
>>       <dependency org="com.spatial4j" name="spatial4j" rev="0.3" transitive="false"/>
>>       <dependency org="javax.servlet" name="javax.servlet-api" rev="3.0.1" transitive="false"/>
>> +      <dependency org="com.yammer.metrics" name="metrics-core" rev="2.1.2" transitive="false"/>
>>       <exclude org="*" ext="*" matcher="regexp" type="${ivy.exclude.types}"/>
>>     </dependencies>
>> </ivy-module>
>> 
>> Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java
>> URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java?rev=1403555&r1=1403554&r2=1403555&view=diff
>> ==============================================================================
>> --- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java
(original)
>> +++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java
Mon Oct 29 22:13:03 2012
>> @@ -17,6 +17,11 @@
>> 
>> package org.apache.solr.handler;
>> 
>> +import com.yammer.metrics.Metrics;
>> +import com.yammer.metrics.core.Counter;
>> +import com.yammer.metrics.core.Timer;
>> +import com.yammer.metrics.core.TimerContext;
>> +import com.yammer.metrics.stats.Snapshot;
>> import org.apache.lucene.queryparser.classic.ParseException;
>> import org.apache.solr.common.SolrException;
>> import org.apache.solr.common.params.SolrParams;
>> @@ -30,26 +35,34 @@ import org.apache.solr.response.SolrQuer
>> import org.apache.solr.util.SolrPluginUtils;
>> 
>> import java.net.URL;
>> +import java.util.concurrent.atomic.AtomicLong;
>> 
>> /**
>>  *
>>  */
>> public abstract class RequestHandlerBase implements SolrRequestHandler, SolrInfoMBean
{
>> 
>> -  // statistics
>> -  // TODO: should we bother synchronizing these, or is an off-by-one error
>> -  // acceptable every million requests or so?
>> -  volatile long numRequests;
>> -  volatile long numErrors;
>> -  volatile long numTimeouts;
>>   protected NamedList initArgs = null;
>>   protected SolrParams defaults;
>>   protected SolrParams appends;
>>   protected SolrParams invariants;
>> -  volatile long totalTime = 0;
>> -  long handlerStart = System.currentTimeMillis();
>>   protected boolean httpCaching = true;
>> 
>> +  // Statistics
>> +  private static final AtomicLong handlerNumber = new AtomicLong();
>> +  private final Counter numRequests;
>> +  private final Counter numErrors;
>> +  private final Counter numTimeouts;
>> +  private final Timer requestTimes;
>> +  long handlerStart = System.currentTimeMillis();
>> +
>> +  public RequestHandlerBase() {
>> +    String scope = new String("metrics-scope-" + handlerNumber.getAndIncrement());
>> +    numRequests = Metrics.newCounter(RequestHandlerBase.class, "numRequests", scope);
>> +    numErrors = Metrics.newCounter(RequestHandlerBase.class, "numErrors", scope);
>> +    numTimeouts = Metrics.newCounter(RequestHandlerBase.class, "numTimeouts", scope);
>> +    requestTimes = Metrics.newTimer(RequestHandlerBase.class, "requestTimes", scope);
>> +  }
>> 
>>   /**
>>    * Initializes the {@link org.apache.solr.request.SolrRequestHandler} by creating
three {@link org.apache.solr.common.params.SolrParams} named.
>> @@ -93,7 +106,7 @@ public abstract class RequestHandlerBase
>>   public void init(NamedList args) {
>>     initArgs = args;
>> 
>> -    // Copied from StandardRequestHandler
>> +    // Copied from StandardRequestHandler
>>     if( args != null ) {
>>       Object o = args.get("defaults");
>>       if (o != null && o instanceof NamedList) {
>> @@ -113,6 +126,7 @@ public abstract class RequestHandlerBase
>>       Object caching = initArgs.get("httpCaching");
>>       httpCaching = caching != null ? Boolean.parseBoolean(caching.toString()) :
true;
>>     }
>> +
>>   }
>> 
>>   public NamedList getInitArgs() {
>> @@ -122,7 +136,8 @@ public abstract class RequestHandlerBase
>>   public abstract void handleRequestBody( SolrQueryRequest req, SolrQueryResponse
rsp ) throws Exception;
>> 
>>   public void handleRequest(SolrQueryRequest req, SolrQueryResponse rsp) {
>> -    numRequests++;
>> +    numRequests.inc();
>> +    TimerContext timer = requestTimes.time();
>>     try {
>>       SolrPluginUtils.setDefaults(req,defaults,appends,invariants);
>>       rsp.setHttpCaching(httpCaching);
>> @@ -133,7 +148,7 @@ public abstract class RequestHandlerBase
>>         Object partialResults = header.get("partialResults");
>>         boolean timedOut = partialResults == null ? false : (Boolean)partialResults;
>>         if( timedOut ) {
>> -          numTimeouts++;
>> +          numTimeouts.inc();
>>           rsp.setHttpCaching(false);
>>         }
>>       }
>> @@ -154,11 +169,12 @@ public abstract class RequestHandlerBase
>>       }
>> 
>>       rsp.setException(e);
>> -      numErrors++;
>> +      numErrors.inc();
>> +    }
>> +    finally {
>> +      timer.stop();
>>     }
>> -    totalTime += rsp.getEndTime() - req.getStartTime();
>>   }
>> -
>> 
>>   //////////////////////// SolrInfoMBeans methods //////////////////////
>> 
>> @@ -184,12 +200,20 @@ public abstract class RequestHandlerBase
>>   public NamedList<Object> getStatistics() {
>>     NamedList<Object> lst = new SimpleOrderedMap<Object>();
>>     lst.add("handlerStart",handlerStart);
>> -    lst.add("requests", numRequests);
>> -    lst.add("errors", numErrors);
>> -    lst.add("timeouts", numTimeouts);
>> -    lst.add("totalTime",totalTime);
>> -    lst.add("avgTimePerRequest", (float) totalTime / (float) this.numRequests);
>> -    lst.add("avgRequestsPerSecond", (float) numRequests*1000 / (float)(System.currentTimeMillis()-handlerStart));
>> +    lst.add("requests", numRequests.count());
>> +    lst.add("errors", numErrors.count());
>> +    lst.add("timeouts", numTimeouts.count());
>> +    lst.add("totalTime",requestTimes.sum());
>> +    lst.add("avgRequestsPerSecond", requestTimes.meanRate());
>> +    lst.add("5minRateReqsPerSecond", requestTimes.fiveMinuteRate());
>> +    lst.add("15minRateReqsPerSecond", requestTimes.fifteenMinuteRate());
>> +    lst.add("avgTimePerRequest", requestTimes.mean());
>> +    Snapshot snapshot = requestTimes.getSnapshot();
>> +    lst.add("medianRequestTime", snapshot.getMedian());
>> +    lst.add("75thPcRequestTime", snapshot.get75thPercentile());
>> +    lst.add("95thPcRequestTime", snapshot.get95thPercentile());
>> +    lst.add("99thPcRequestTime", snapshot.get99thPercentile());
>> +    lst.add("999thPcRequestTime", snapshot.get999thPercentile());
>>     return lst;
>>   }
>> 
>> 
>> Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/core/RequestHandlersTest.java
>> URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/core/RequestHandlersTest.java?rev=1403555&r1=1403554&r2=1403555&view=diff
>> ==============================================================================
>> --- lucene/dev/trunk/solr/core/src/test/org/apache/solr/core/RequestHandlersTest.java
(original)
>> +++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/core/RequestHandlersTest.java
Mon Oct 29 22:13:03 2012
>> @@ -18,6 +18,7 @@
>> package org.apache.solr.core;
>> 
>> import org.apache.solr.SolrTestCaseJ4;
>> +import org.apache.solr.common.util.NamedList;
>> import org.apache.solr.handler.StandardRequestHandler;
>> import org.apache.solr.request.SolrRequestHandler;
>> import org.junit.BeforeClass;
>> @@ -87,4 +88,23 @@ public class RequestHandlersTest extends
>> 
>>     assertNull( core.getRequestHandler("/update/asdgadsgas" ) ); // prefix
>>   }
>> +
>> +  @Test
>> +  public void testStatistics() {
>> +    SolrCore core = h.getCore();
>> +    SolrRequestHandler updateHandler = core.getRequestHandler("/update");
>> +    SolrRequestHandler termHandler = core.getRequestHandler("/terms");
>> +
>> +    assertU(adoc("id", "47",
>> +        "text", "line up and fly directly at the enemy death cannons, clogging them
with wreckage!"));
>> +    assertU(commit());
>> +
>> +    NamedList updateStats = updateHandler.getStatistics();
>> +    NamedList termStats = termHandler.getStatistics();
>> +
>> +    Double updateTime = (Double) updateStats.get("totalTime");
>> +    Double termTime = (Double) termStats.get("totalTime");
>> +
>> +    assertFalse("RequestHandlers should not share statistics!", updateTime.equals(termTime));
>> +  }
>> }
>> 
>> Modified: lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/SolrIgnoredThreadsFilter.java
>> URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/SolrIgnoredThreadsFilter.java?rev=1403555&r1=1403554&r2=1403555&view=diff
>> ==============================================================================
>> --- lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/SolrIgnoredThreadsFilter.java
(original)
>> +++ lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/SolrIgnoredThreadsFilter.java
Mon Oct 29 22:13:03 2012
>> @@ -42,7 +42,8 @@ public class SolrIgnoredThreadsFilter im
>>       return true;
>>     }
>> 
>> -    if (threadName.startsWith("facetExecutor-") ||
>> +    if (threadName.startsWith("facetExecutor-") ||
>> +        threadName.startsWith("metrics-") ||
>>         threadName.startsWith("cmdDistribExecutor-") ||
>>         threadName.startsWith("httpShardExecutor-")) {
>>       return true;
>> 
>> 
> 
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org
> 


---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org


Mime
View raw message