lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From markrmil...@apache.org
Subject lucene-solr:master: SOLR-12290: We must close ContentStreams because we don't know the source of the inputstream - use a CloseShield to prevent tripping our close assert in SolrDispatchFilter.
Date Sun, 06 May 2018 19:33:57 GMT
Repository: lucene-solr
Updated Branches:
  refs/heads/master 0922e58c2 -> 5fc725154


SOLR-12290: We must close ContentStreams because we don't know the source of the inputstream
- use a CloseShield to prevent tripping our close assert in SolrDispatchFilter.


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/5fc72515
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/5fc72515
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/5fc72515

Branch: refs/heads/master
Commit: 5fc725154001c6283315802e1a2193d51d00f9aa
Parents: 0922e58
Author: markrmiller <markrmiller@apache.org>
Authored: Sun May 6 14:25:59 2018 -0500
Committer: markrmiller <markrmiller@apache.org>
Committed: Sun May 6 14:25:59 2018 -0500

----------------------------------------------------------------------
 .../org/apache/solr/handler/BlobHandler.java    |  6 +-
 .../solr/handler/loader/CSVLoaderBase.java      | 75 +++++++++++---------
 .../solr/handler/loader/JavabinLoader.java      | 16 ++++-
 .../apache/solr/servlet/SolrRequestParsers.java |  6 +-
 4 files changed, 64 insertions(+), 39 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5fc72515/solr/core/src/java/org/apache/solr/handler/BlobHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/BlobHandler.java b/solr/core/src/java/org/apache/solr/handler/BlobHandler.java
index a998657..30301c0 100644
--- a/solr/core/src/java/org/apache/solr/handler/BlobHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/BlobHandler.java
@@ -17,6 +17,7 @@
 package org.apache.solr.handler;
 
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.OutputStream;
 import java.lang.invoke.MethodHandles;
 import java.math.BigInteger;
@@ -108,8 +109,9 @@ public class BlobHandler extends RequestHandlerBase implements PluginInfoInitial
 
       for (ContentStream stream : req.getContentStreams()) {
         ByteBuffer payload;
-        payload = SimplePostTool.inputStreamToByteArray(stream.getStream(), maxSize);
-        
+        try (InputStream is = stream.getStream()) {
+          payload = SimplePostTool.inputStreamToByteArray(is, maxSize);
+        }
         MessageDigest m = MessageDigest.getInstance("MD5");
         m.update(payload.array(), payload.position(), payload.limit());
         String md5 = new BigInteger(1, m.digest()).toString(16);

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5fc72515/solr/core/src/java/org/apache/solr/handler/loader/CSVLoaderBase.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/loader/CSVLoaderBase.java b/solr/core/src/java/org/apache/solr/handler/loader/CSVLoaderBase.java
index fd8935d..b503fa3 100644
--- a/solr/core/src/java/org/apache/solr/handler/loader/CSVLoaderBase.java
+++ b/solr/core/src/java/org/apache/solr/handler/loader/CSVLoaderBase.java
@@ -28,6 +28,7 @@ import org.apache.solr.update.*;
 import org.apache.solr.update.processor.UpdateRequestProcessor;
 import org.apache.solr.internal.csv.CSVStrategy;
 import org.apache.solr.internal.csv.CSVParser;
+import org.apache.commons.io.IOUtils;
 
 import java.util.regex.Pattern;
 import java.util.List;
@@ -314,48 +315,54 @@ abstract class CSVLoaderBase extends ContentStreamLoader {
 
   /** load the CSV input */
   @Override
-  public void load(SolrQueryRequest req, SolrQueryResponse rsp, ContentStream stream, UpdateRequestProcessor
processor)
-      throws IOException {
+  public void load(SolrQueryRequest req, SolrQueryResponse rsp, ContentStream stream, UpdateRequestProcessor
processor) throws IOException {
     errHeader = "CSVLoader: input=" + stream.getSourceInfo();
-    Reader reader = stream.getReader();
-    if (skipLines > 0) {
-      if (!(reader instanceof BufferedReader)) {
-        reader = new BufferedReader(reader);
-      }
-      BufferedReader r = (BufferedReader) reader;
-      for (int i = 0; i < skipLines; i++) {
-        r.readLine();
+    Reader reader = null;
+    try {
+      reader = stream.getReader();
+      if (skipLines>0) {
+        if (!(reader instanceof BufferedReader)) {
+          reader = new BufferedReader(reader);
+        }
+        BufferedReader r = (BufferedReader)reader;
+        for (int i=0; i<skipLines; i++) {
+          r.readLine();
+        }
       }
-    }
 
-    CSVParser parser = new CSVParser(reader, strategy);
+      CSVParser parser = new CSVParser(reader, strategy);
 
-    // parse the fieldnames from the header of the file
-    if (fieldnames == null) {
-      fieldnames = parser.getLine();
-      if (fieldnames == null) {
-        throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Expected fieldnames
in CSV input");
+      // parse the fieldnames from the header of the file
+      if (fieldnames==null) {
+        fieldnames = parser.getLine();
+        if (fieldnames==null) {
+          throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Expected fieldnames
in CSV input");
+        }
+        prepareFields();
       }
-      prepareFields();
-    }
 
-    // read the rest of the CSV file
-    for (;;) {
-      int line = parser.getLineNumber(); // for error reporting in MT mode
-      String[] vals = null;
-      try {
-        vals = parser.getLine();
-      } catch (IOException e) {
-        // Catch the exception and rethrow it with more line information
-        input_err("can't read line: " + line, null, line, e);
-      }
-      if (vals == null) break;
+      // read the rest of the CSV file
+      for(;;) {
+        int line = parser.getLineNumber();  // for error reporting in MT mode
+        String[] vals = null;
+        try {
+          vals = parser.getLine();
+        } catch (IOException e) {
+          //Catch the exception and rethrow it with more line information
+         input_err("can't read line: " + line, null, line, e);
+        }
+        if (vals==null) break;
 
-      if (vals.length != fieldnames.length) {
-        input_err("expected " + fieldnames.length + " values but got " + vals.length, vals,
line);
-      }
+        if (vals.length != fieldnames.length) {
+          input_err("expected "+fieldnames.length+" values but got "+vals.length, vals, line);
+        }
 
-      addDoc(line, vals);
+        addDoc(line,vals);
+      }
+    } finally{
+      if (reader != null) {
+        IOUtils.closeQuietly(reader);
+      }
     }
   }
 

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5fc72515/solr/core/src/java/org/apache/solr/handler/loader/JavabinLoader.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/loader/JavabinLoader.java b/solr/core/src/java/org/apache/solr/handler/loader/JavabinLoader.java
index aca3df4..f502a8e 100644
--- a/solr/core/src/java/org/apache/solr/handler/loader/JavabinLoader.java
+++ b/solr/core/src/java/org/apache/solr/handler/loader/JavabinLoader.java
@@ -48,8 +48,20 @@ import org.apache.solr.update.processor.UpdateRequestProcessor;
 public class JavabinLoader extends ContentStreamLoader {
 
   @Override
-  public void load(SolrQueryRequest req, SolrQueryResponse rsp, ContentStream cs, UpdateRequestProcessor
processor) throws Exception {
-    InputStream stream = cs.getStream();
+  public void load(SolrQueryRequest req, SolrQueryResponse rsp, ContentStream stream, UpdateRequestProcessor
processor) throws Exception {
+    InputStream is = null;
+    try {
+      is = stream.getStream();
+      parseAndLoadDocs(req, rsp, is, processor);
+    } finally {
+      if(is != null) {
+        is.close();
+      }
+    }
+  }
+  
+  private void parseAndLoadDocs(final SolrQueryRequest req, SolrQueryResponse rsp, InputStream
stream,
+                                final UpdateRequestProcessor processor) throws IOException
{
     UpdateRequest update = null;
     JavaBinUpdateRequestCodec.StreamingUpdateHandler handler = new JavaBinUpdateRequestCodec.StreamingUpdateHandler()
{
       private AddUpdateCommand addCmd = null;

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5fc72515/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java b/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java
index 4bcb8d8..8a664c4 100644
--- a/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java
+++ b/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java
@@ -519,7 +519,11 @@ public class SolrRequestParsers
 
     @Override
     public InputStream getStream() throws IOException {
-      return req.getInputStream();
+      // we explicitly protect this servlet stream from being closed
+      // so that it does not trip our test assert in our close shield
+      // in SolrDispatchFilter - we must allow closes from getStream
+      // due to the other impls of ContentStream
+      return new CloseShieldInputStream(req.getInputStream());
     }
   }
 


Mime
View raw message