lucene-solr-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From k...@apache.org
Subject svn commit: r949651 - in /lucene/solr/branches/branch-1.4: ./ src/java/org/apache/solr/handler/ src/java/org/apache/solr/request/ src/solrj/org/apache/solr/client/solrj/impl/
Date Mon, 31 May 2010 02:11:06 GMT
Author: koji
Date: Mon May 31 02:11:05 2010
New Revision: 949651

URL: http://svn.apache.org/viewvc?rev=949651&view=rev
Log:
merging r906553 from solr/branches/branch-1.5-dev for SOLR-1748, SOLR-1747, SOLR-1746, SOLR-1745
and SOLR-1744: Streams and Readers retrieved from ContentStreams are not closed in various
places, resulting in file descriptor leaks.

Modified:
    lucene/solr/branches/branch-1.4/CHANGES.txt
    lucene/solr/branches/branch-1.4/src/java/org/apache/solr/handler/BinaryUpdateRequestHandler.java
  (contents, props changed)
    lucene/solr/branches/branch-1.4/src/java/org/apache/solr/handler/DumpRequestHandler.java
  (contents, props changed)
    lucene/solr/branches/branch-1.4/src/java/org/apache/solr/handler/MoreLikeThisHandler.java
  (contents, props changed)
    lucene/solr/branches/branch-1.4/src/java/org/apache/solr/request/RawResponseWriter.java
  (contents, props changed)
    lucene/solr/branches/branch-1.4/src/solrj/org/apache/solr/client/solrj/impl/CommonsHttpSolrServer.java
  (contents, props changed)

Modified: lucene/solr/branches/branch-1.4/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/solr/branches/branch-1.4/CHANGES.txt?rev=949651&r1=949650&r2=949651&view=diff
==============================================================================
--- lucene/solr/branches/branch-1.4/CHANGES.txt (original)
+++ lucene/solr/branches/branch-1.4/CHANGES.txt Mon May 31 02:11:05 2010
@@ -67,6 +67,11 @@ Bug Fixes
 
 * SOLR-1711: SolrJ - StreamingUpdateSolrServer had a race condition that
   could halt the streaming of documents. (Attila Babo via yonik)
+  
+* SOLR-1748, SOLR-1747, SOLR-1746, SOLR-1745, SOLR-1744: Streams and Readers
+  retrieved from ContentStreams are not closed in various places, resulting
+  in file descriptor leaks.
+  (Christoff Brill, Mark Miller)
 
 * SOLR-1777: fieldTypes with sortMissingLast=true or sortMissingFirst=true can
   result in incorrectly sorted results.  (yonik)

Modified: lucene/solr/branches/branch-1.4/src/java/org/apache/solr/handler/BinaryUpdateRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/branch-1.4/src/java/org/apache/solr/handler/BinaryUpdateRequestHandler.java?rev=949651&r1=949650&r2=949651&view=diff
==============================================================================
--- lucene/solr/branches/branch-1.4/src/java/org/apache/solr/handler/BinaryUpdateRequestHandler.java
(original)
+++ lucene/solr/branches/branch-1.4/src/java/org/apache/solr/handler/BinaryUpdateRequestHandler.java
Mon May 31 02:11:05 2010
@@ -49,7 +49,15 @@ public class BinaryUpdateRequestHandler 
   protected ContentStreamLoader newLoader(SolrQueryRequest req, final UpdateRequestProcessor
processor) {
     return new ContentStreamLoader() {
       public void load(SolrQueryRequest req, SolrQueryResponse rsp, ContentStream stream)
throws Exception {
-        parseAndLoadDocs(req, rsp, stream.getStream(), processor);
+        InputStream is = null;
+        try {
+          is = stream.getStream();
+          parseAndLoadDocs(req, rsp, is, processor);
+        } finally {
+          if(is != null) {
+            is.close();
+          }
+        }
       }
     };
   }

Propchange: lucene/solr/branches/branch-1.4/src/java/org/apache/solr/handler/BinaryUpdateRequestHandler.java
------------------------------------------------------------------------------
--- svn:mergeinfo (added)
+++ svn:mergeinfo Mon May 31 02:11:05 2010
@@ -0,0 +1 @@
+/lucene/solr/trunk/src/java/org/apache/solr/handler/BinaryUpdateRequestHandler.java:881642,881906,883566,891596,891889,903271,906553,911245,919871,925898

Modified: lucene/solr/branches/branch-1.4/src/java/org/apache/solr/handler/DumpRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/branch-1.4/src/java/org/apache/solr/handler/DumpRequestHandler.java?rev=949651&r1=949650&r2=949651&view=diff
==============================================================================
--- lucene/solr/branches/branch-1.4/src/java/org/apache/solr/handler/DumpRequestHandler.java
(original)
+++ lucene/solr/branches/branch-1.4/src/java/org/apache/solr/handler/DumpRequestHandler.java
Mon May 31 02:11:05 2010
@@ -18,6 +18,7 @@
 package org.apache.solr.handler;
 
 import java.io.IOException;
+import java.io.InputStream;
 import java.util.ArrayList;
 
 import org.apache.commons.io.IOUtils;
@@ -45,7 +46,12 @@ public class DumpRequestHandler extends 
         stream.add( "sourceInfo", content.getSourceInfo() );
         stream.add( "size", content.getSize() );
         stream.add( "contentType", content.getContentType() );
-        stream.add( "stream", IOUtils.toString( content.getStream() ) );
+        InputStream is = content.getStream();
+        try {
+          stream.add( "stream", IOUtils.toString(is) );
+        } finally {
+          is.close();
+        }
         streams.add( stream );
       }
       rsp.add( "streams", streams );

Propchange: lucene/solr/branches/branch-1.4/src/java/org/apache/solr/handler/DumpRequestHandler.java
------------------------------------------------------------------------------
--- svn:mergeinfo (added)
+++ svn:mergeinfo Mon May 31 02:11:05 2010
@@ -0,0 +1 @@
+/lucene/solr/trunk/src/java/org/apache/solr/handler/DumpRequestHandler.java:881642,881906,883566,891596,891889,903271,906553,911245,919871,925898

Modified: lucene/solr/branches/branch-1.4/src/java/org/apache/solr/handler/MoreLikeThisHandler.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/branch-1.4/src/java/org/apache/solr/handler/MoreLikeThisHandler.java?rev=949651&r1=949650&r2=949651&view=diff
==============================================================================
--- lucene/solr/branches/branch-1.4/src/java/org/apache/solr/handler/MoreLikeThisHandler.java
(original)
+++ lucene/solr/branches/branch-1.4/src/java/org/apache/solr/handler/MoreLikeThisHandler.java
Mon May 31 02:11:05 2010
@@ -23,11 +23,9 @@ import java.net.MalformedURLException;
 import java.net.URL;
 import java.util.ArrayList;
 import java.util.Comparator;
-import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 import java.util.regex.Pattern;
 
 import org.apache.lucene.document.Document;
@@ -40,7 +38,6 @@ import org.apache.lucene.search.TermQuer
 import org.apache.lucene.search.similar.MoreLikeThis;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.params.CommonParams;
-import org.apache.solr.common.params.DisMaxParams;
 import org.apache.solr.common.params.FacetParams;
 import org.apache.solr.common.params.MoreLikeThisParams;
 import org.apache.solr.common.params.SolrParams;
@@ -84,24 +81,7 @@ public class MoreLikeThisHandler extends
     SolrParams params = req.getParams();
     SolrIndexSearcher searcher = req.getSearcher();
     
-    // Parse Required Params
-    // This will either have a single Reader or valid query
-    Reader reader = null;
-    String q = params.get( CommonParams.Q );
-    if( q == null || q.trim().length() <1 ) {
-      Iterable<ContentStream> streams = req.getContentStreams();
-      if( streams != null ) {
-        Iterator<ContentStream> iter = streams.iterator();
-        if( iter.hasNext() ) {
-          reader = iter.next().getReader();
-        }
-        if( iter.hasNext() ) {
-          throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, 
-              "MoreLikeThis does not support multiple ContentStreams" );
-        }
-      }
-    }
-
+    
     MoreLikeThisHelper mlt = new MoreLikeThisHelper( params, searcher );
     List<Query> filters = SolrPluginUtils.parseFilterQueries(req);
     
@@ -110,46 +90,75 @@ public class MoreLikeThisHandler extends
     List<InterestingTerm> interesting = (termStyle == TermStyle.NONE )
       ? null : new ArrayList<InterestingTerm>( mlt.mlt.getMaxQueryTerms() );
     
-    // What fields do we need to return
-    String fl = params.get(CommonParams.FL);
-    int flags = 0; 
-    if (fl != null) {
-      flags |= SolrPluginUtils.setReturnFields(fl, rsp);
-    }
-
-    int start = params.getInt( CommonParams.START, 0 );
-    int rows  = params.getInt( CommonParams.ROWS, 10 );
-    
     DocListAndSet mltDocs = null;
+    String q = params.get( CommonParams.Q );
     
-    // Find documents MoreLikeThis - either with a reader or a query
-    //--------------------------------------------------------------------------------
-    if( reader != null ) {
-      mltDocs = mlt.getMoreLikeThis( reader, start, rows, filters, interesting, flags );
-    }
-    else if( q != null ) {
-      // Matching options
-      boolean includeMatch = params.getBool( MoreLikeThisParams.MATCH_INCLUDE, true );
-      int matchOffset = params.getInt( MoreLikeThisParams.MATCH_OFFSET, 0 );
-      // Find the base match  
-      Query query = QueryParsing.parseQuery(q, params.get(CommonParams.DF), params, req.getSchema());
-      DocList match = searcher.getDocList(query, null, null, matchOffset, 1, flags ); //
only get the first one...
-      if( includeMatch ) {
-        rsp.add( "match", match );
-      }
-
-      // This is an iterator, but we only handle the first match
-      DocIterator iterator = match.iterator();
-      if( iterator.hasNext() ) {
-        // do a MoreLikeThis query for each document in results
-        int id = iterator.nextDoc();
-        mltDocs = mlt.getMoreLikeThis( id, start, rows, filters, interesting, flags );
+    // Parse Required Params
+    // This will either have a single Reader or valid query
+    Reader reader = null;
+    try {
+      if (q == null || q.trim().length() < 1) {
+        Iterable<ContentStream> streams = req.getContentStreams();
+        if (streams != null) {
+          Iterator<ContentStream> iter = streams.iterator();
+          if (iter.hasNext()) {
+            reader = iter.next().getReader();
+          }
+          if (iter.hasNext()) {
+            throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+                "MoreLikeThis does not support multiple ContentStreams");
+          }
+        }
+      }
+
+      // What fields do we need to return
+      String fl = params.get(CommonParams.FL);
+      int flags = 0;
+      if (fl != null) {
+        flags |= SolrPluginUtils.setReturnFields(fl, rsp);
+      }
+
+      int start = params.getInt(CommonParams.START, 0);
+      int rows = params.getInt(CommonParams.ROWS, 10);
+
+      // Find documents MoreLikeThis - either with a reader or a query
+      // --------------------------------------------------------------------------------
+      if (reader != null) {
+        mltDocs = mlt.getMoreLikeThis(reader, start, rows, filters,
+            interesting, flags);
+      } else if (q != null) {
+        // Matching options
+        boolean includeMatch = params.getBool(MoreLikeThisParams.MATCH_INCLUDE,
+            true);
+        int matchOffset = params.getInt(MoreLikeThisParams.MATCH_OFFSET, 0);
+        // Find the base match
+        Query query = QueryParsing.parseQuery(q, params.get(CommonParams.DF),
+            params, req.getSchema());
+        DocList match = searcher.getDocList(query, null, null, matchOffset, 1,
+            flags); // only get the first one...
+        if (includeMatch) {
+          rsp.add("match", match);
+        }
+
+        // This is an iterator, but we only handle the first match
+        DocIterator iterator = match.iterator();
+        if (iterator.hasNext()) {
+          // do a MoreLikeThis query for each document in results
+          int id = iterator.nextDoc();
+          mltDocs = mlt.getMoreLikeThis(id, start, rows, filters, interesting,
+              flags);
+        }
+      } else {
+        throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+            "MoreLikeThis requires either a query (?q=) or text to find similar documents.");
+      }
+
+    } finally {
+      if (reader != null) {
+        reader.close();
       }
     }
-    else {
-      throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, 
-          "MoreLikeThis requires either a query (?q=) or text to find similar documents."
);
-    }
+    
     if( mltDocs == null ) {
       mltDocs = new DocListAndSet(); // avoid NPE
     }

Propchange: lucene/solr/branches/branch-1.4/src/java/org/apache/solr/handler/MoreLikeThisHandler.java
------------------------------------------------------------------------------
--- svn:mergeinfo (added)
+++ svn:mergeinfo Mon May 31 02:11:05 2010
@@ -0,0 +1 @@
+/lucene/solr/trunk/src/java/org/apache/solr/handler/MoreLikeThisHandler.java:881642,881906,883566,891596,891889,903271,906553,911245,919871,925898

Modified: lucene/solr/branches/branch-1.4/src/java/org/apache/solr/request/RawResponseWriter.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/branch-1.4/src/java/org/apache/solr/request/RawResponseWriter.java?rev=949651&r1=949650&r2=949651&view=diff
==============================================================================
--- lucene/solr/branches/branch-1.4/src/java/org/apache/solr/request/RawResponseWriter.java
(original)
+++ lucene/solr/branches/branch-1.4/src/java/org/apache/solr/request/RawResponseWriter.java
Mon May 31 02:11:05 2010
@@ -18,6 +18,7 @@
 package org.apache.solr.request;
 
 import java.io.IOException;
+import java.io.Reader;
 import java.io.Writer;
 
 import org.apache.commons.io.IOUtils;
@@ -80,7 +81,12 @@ public class RawResponseWriter implement
     if( obj != null && (obj instanceof ContentStream ) ) {
       // copy the contents to the writer...
       ContentStream content = (ContentStream)obj;
-      IOUtils.copy( content.getReader(), writer );
+      Reader reader = content.getReader();
+      try {
+        IOUtils.copy( reader, writer );
+      } finally {
+        reader.close();
+      }
     }
     else {
       getBaseWriter( request ).write( writer, request, response );

Propchange: lucene/solr/branches/branch-1.4/src/java/org/apache/solr/request/RawResponseWriter.java
------------------------------------------------------------------------------
--- svn:mergeinfo (added)
+++ svn:mergeinfo Mon May 31 02:11:05 2010
@@ -0,0 +1,2 @@
+/lucene/solr/trunk/src/java/org/apache/solr/request/RawResponseWriter.java:881642,881906,883566,891596,891889,903271,911245,919871,925898
+/lucene/solr/trunk/src/java/org/apache/solr/response/RawResponseWriter.java:906553

Modified: lucene/solr/branches/branch-1.4/src/solrj/org/apache/solr/client/solrj/impl/CommonsHttpSolrServer.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/branch-1.4/src/solrj/org/apache/solr/client/solrj/impl/CommonsHttpSolrServer.java?rev=949651&r1=949650&r2=949651&view=diff
==============================================================================
--- lucene/solr/branches/branch-1.4/src/solrj/org/apache/solr/client/solrj/impl/CommonsHttpSolrServer.java
(original)
+++ lucene/solr/branches/branch-1.4/src/solrj/org/apache/solr/client/solrj/impl/CommonsHttpSolrServer.java
Mon May 31 02:11:05 2010
@@ -20,6 +20,7 @@ package org.apache.solr.client.solrj.imp
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.io.Reader;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.util.*;
@@ -246,6 +247,7 @@ public class CommonsHttpSolrServer exten
   
   public NamedList<Object> request(final SolrRequest request, ResponseParser processor)
throws SolrServerException, IOException {
     HttpMethod method = null;
+    InputStream is = null;
     SolrParams params = request.getParams();
     Collection<ContentStream> streams = requestWriter.getContentStreams(request);
     String path = requestWriter.getPath(request);
@@ -333,7 +335,12 @@ public class CommonsHttpSolrServer exten
                     @Override
                     protected void sendData(OutputStream out)
                         throws IOException {
-                      IOUtils.copy(c.getReader(), out);
+                      Reader reader = c.getReader();
+                      try {
+                        IOUtils.copy(reader, out);
+                      } finally {
+                        reader.close();
+                      }
                     }
                   });
                 }
@@ -378,7 +385,8 @@ public class CommonsHttpSolrServer exten
                 );
 
               } else {
-                post.setRequestEntity(new InputStreamRequestEntity(contentStream[0].getStream(),
contentStream[0].getContentType()));
+                is = contentStream[0].getStream();
+                post.setRequestEntity(new InputStreamRequestEntity(is, contentStream[0].getContentType()));
               }
               method = post;
             }
@@ -391,6 +399,9 @@ public class CommonsHttpSolrServer exten
           // This is generally safe to retry on
           method.releaseConnection();
           method = null;
+          if(is != null) {
+            is.close();
+          }
           // If out of tries then just rethrow (as normal error).
           if( ( tries < 1 ) ) {
             throw r;
@@ -473,6 +484,9 @@ public class CommonsHttpSolrServer exten
     }
     finally {
       method.releaseConnection();
+      if(is != null) {
+        is.close();
+      }
     }
   }
 

Propchange: lucene/solr/branches/branch-1.4/src/solrj/org/apache/solr/client/solrj/impl/CommonsHttpSolrServer.java
------------------------------------------------------------------------------
--- svn:mergeinfo (added)
+++ svn:mergeinfo Mon May 31 02:11:05 2010
@@ -0,0 +1 @@
+/lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/impl/CommonsHttpSolrServer.java:829939-906553,911245,919871,925898



Mime
View raw message