lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From c.@apache.org
Subject svn commit: r1512909 [29/38] - in /lucene/dev/branches/lucene4956: ./ dev-tools/ dev-tools/eclipse/ dev-tools/idea/.idea/libraries/ dev-tools/idea/lucene/suggest/ dev-tools/idea/solr/contrib/dataimporthandler/ dev-tools/idea/solr/core/src/test/ dev-too...
Date Sun, 11 Aug 2013 12:19:39 GMT
Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java Sun Aug 11 12:19:13 2013
@@ -28,9 +28,11 @@ import java.nio.charset.CharsetDecoder;
 import java.nio.charset.CodingErrorAction;
 import java.net.URL;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.Iterator;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
@@ -68,6 +70,11 @@ public class SolrRequestParsers 
   public static final String SIMPLE = "simple";
   public static final String STANDARD = "standard";
   
+  private static final Charset CHARSET_US_ASCII = Charset.forName("US-ASCII");
+  
+  public static final String INPUT_ENCODING_KEY = "ie";
+  private static final byte[] INPUT_ENCODING_BYTES = INPUT_ENCODING_KEY.getBytes(CHARSET_US_ASCII);
+
   private final HashMap<String, SolrRequestParser> parsers =
       new HashMap<String, SolrRequestParser>();
   private final boolean enableRemoteStreams;
@@ -242,7 +249,7 @@ public class SolrRequestParsers 
             }
           }
         };
-        parseFormDataContent(in, Long.MAX_VALUE, IOUtils.CHARSET_UTF_8, map);
+        parseFormDataContent(in, Long.MAX_VALUE, IOUtils.CHARSET_UTF_8, map, true);
       } catch (IOException ioe) {
         throw new SolrException(ErrorCode.BAD_REQUEST, ioe);
       }
@@ -256,23 +263,53 @@ public class SolrRequestParsers 
    * @param charset to be used to decode resulting bytes after %-decoding
    * @param map place all parameters in this map
    */
-  @SuppressWarnings("fallthrough")
-  static long parseFormDataContent(final InputStream postContent, final long maxLen, final Charset charset, final Map<String,String[]> map) throws IOException {
-    final CharsetDecoder charsetDecoder = charset.newDecoder()
-      .onMalformedInput(CodingErrorAction.REPORT)
-      .onUnmappableCharacter(CodingErrorAction.REPORT);
+  @SuppressWarnings({"fallthrough", "resource"})
+  static long parseFormDataContent(final InputStream postContent, final long maxLen, Charset charset, final Map<String,String[]> map, boolean supportCharsetParam) throws IOException {
+    CharsetDecoder charsetDecoder = supportCharsetParam ? null : getCharsetDecoder(charset);
+    final LinkedList<Object> buffer = supportCharsetParam ? new LinkedList<Object>() : null;
     long len = 0L, keyPos = 0L, valuePos = 0L;
-    final ByteArrayOutputStream2 keyStream = new ByteArrayOutputStream2(),
-      valueStream = new ByteArrayOutputStream2();
-    ByteArrayOutputStream2 currentStream = keyStream;
+    final ByteArrayOutputStream keyStream = new ByteArrayOutputStream(),
+      valueStream = new ByteArrayOutputStream();
+    ByteArrayOutputStream currentStream = keyStream;
     for(;;) {
       int b = postContent.read();
       switch (b) {
         case -1: // end of stream
         case '&': // separator
           if (keyStream.size() > 0) {
-            final String key = decodeChars(keyStream, keyPos, charsetDecoder), value = decodeChars(valueStream, valuePos, charsetDecoder);
-            MultiMapSolrParams.addParam(key, value, map);
+            final byte[] keyBytes = keyStream.toByteArray(), valueBytes = valueStream.toByteArray();
+            if (Arrays.equals(keyBytes, INPUT_ENCODING_BYTES)) {
+              // we found a charset declaration in the raw bytes
+              if (charsetDecoder != null) {
+                throw new SolrException(ErrorCode.BAD_REQUEST,
+                  supportCharsetParam ? (
+                    "Query string invalid: duplicate '"+
+                    INPUT_ENCODING_KEY + "' (input encoding) key."
+                  ) : (
+                    "Key '" + INPUT_ENCODING_KEY + "' (input encoding) cannot "+
+                    "be used in POSTed application/x-www-form-urlencoded form data. "+
+                    "To set the input encoding of POSTed form data, use the "+
+                    "'Content-Type' header and provide a charset!"
+                  )
+                );
+              }
+              // decode the charset from raw bytes
+              charset = Charset.forName(decodeChars(valueBytes, keyPos, getCharsetDecoder(CHARSET_US_ASCII)));
+              charsetDecoder = getCharsetDecoder(charset);
+              // finally decode all buffered tokens
+              decodeBuffer(buffer, map, charsetDecoder);
+            } else if (charsetDecoder == null) {
+              // we have no charset decoder until now, buffer the keys / values for later processing:
+              buffer.add(keyBytes);
+              buffer.add(Long.valueOf(keyPos));
+              buffer.add(valueBytes);
+              buffer.add(Long.valueOf(valuePos));
+            } else {
+              // we already have a charsetDecoder, so we can directly decode without buffering:
+              final String key = decodeChars(keyBytes, keyPos, charsetDecoder),
+                  value = decodeChars(valueBytes, valuePos, charsetDecoder);
+              MultiMapSolrParams.addParam(key, value, map);
+            }
           } else if (valueStream.size() > 0) {
             throw new SolrException(ErrorCode.BAD_REQUEST, "application/x-www-form-urlencoded invalid: missing key");
           }
@@ -309,12 +346,23 @@ public class SolrRequestParsers 
         throw new SolrException(ErrorCode.BAD_REQUEST, "application/x-www-form-urlencoded content exceeds upload limit of " + (maxLen/1024L) + " KB");
       }
     }
+    // if we have not seen a charset declaration, decode the buffer now using the default one (UTF-8 or given via Content-Type):
+    if (buffer != null && !buffer.isEmpty()) {
+      assert charsetDecoder == null;
+      decodeBuffer(buffer, map, getCharsetDecoder(charset));
+    }
     return len;
   }
   
-  private static String decodeChars(ByteArrayOutputStream2 stream, long position, CharsetDecoder charsetDecoder) {
+  private static CharsetDecoder getCharsetDecoder(Charset charset) {
+    return charset.newDecoder()
+      .onMalformedInput(CodingErrorAction.REPORT)
+      .onUnmappableCharacter(CodingErrorAction.REPORT);
+  }
+  
+  private static String decodeChars(byte[] bytes, long position, CharsetDecoder charsetDecoder) {
     try {
-      return charsetDecoder.decode(ByteBuffer.wrap(stream.buffer(), 0, stream.size())).toString();
+      return charsetDecoder.decode(ByteBuffer.wrap(bytes)).toString();
     } catch (CharacterCodingException cce) {
       throw new SolrException(ErrorCode.BAD_REQUEST,
         "URLDecoder: Invalid character encoding detected after position " + position +
@@ -323,10 +371,18 @@ public class SolrRequestParsers 
     }
   }
   
-  /** Makes the buffer of ByteArrayOutputStream available without copy. */
-  static final class ByteArrayOutputStream2 extends ByteArrayOutputStream {
-    byte[] buffer() {
-      return buf;
+  private static void decodeBuffer(final LinkedList<Object> input, final Map<String,String[]> map, CharsetDecoder charsetDecoder) {
+    for (final Iterator<Object> it = input.iterator(); it.hasNext(); ) {
+      final byte[] keyBytes = (byte[]) it.next();
+      it.remove();
+      final Long keyPos = (Long) it.next();
+      it.remove();
+      final byte[] valueBytes = (byte[]) it.next();
+      it.remove();
+      final Long valuePos = (Long) it.next();
+      it.remove();
+      MultiMapSolrParams.addParam(decodeChars(keyBytes, keyPos.longValue(), charsetDecoder),
+          decodeChars(valueBytes, valuePos.longValue(), charsetDecoder), map);
     }
   }
   
@@ -361,281 +417,272 @@ public class SolrRequestParsers 
   public void setAddRequestHeadersToContext(boolean addRequestHeadersToContext) {
     this.addHttpRequestToContext = addRequestHeadersToContext;
   }
-}
 
-//-----------------------------------------------------------------
-//-----------------------------------------------------------------
+  //-----------------------------------------------------------------
+  //-----------------------------------------------------------------
 
-// I guess we don't really even need the interface, but i'll keep it here just for kicks
-interface SolrRequestParser 
-{
-  public SolrParams parseParamsAndFillStreams(
-    final HttpServletRequest req, ArrayList<ContentStream> streams ) throws Exception;
-}
+  // I guess we don't really even need the interface, but i'll keep it here just for kicks
+  interface SolrRequestParser 
+  {
+    public SolrParams parseParamsAndFillStreams(
+      final HttpServletRequest req, ArrayList<ContentStream> streams ) throws Exception;
+  }
 
 
-//-----------------------------------------------------------------
-//-----------------------------------------------------------------
-
-/**
- * The simple parser just uses the params directly, does not support POST URL-encoded forms
- */
-class SimpleRequestParser implements SolrRequestParser
-{
-  @Override
-  public SolrParams parseParamsAndFillStreams( 
-      final HttpServletRequest req, ArrayList<ContentStream> streams ) throws Exception
+  //-----------------------------------------------------------------
+  //-----------------------------------------------------------------
+
+  /**
+   * The simple parser just uses the params directly, does not support POST URL-encoded forms
+   */
+  static class SimpleRequestParser implements SolrRequestParser
   {
-    return SolrRequestParsers.parseQueryString(req.getQueryString());
+    @Override
+    public SolrParams parseParamsAndFillStreams( 
+        final HttpServletRequest req, ArrayList<ContentStream> streams ) throws Exception
+    {
+      return parseQueryString(req.getQueryString());
+    }
   }
-}
 
-/**
- * Wrap an HttpServletRequest as a ContentStream
- */
-class HttpRequestContentStream extends ContentStreamBase
-{
-  private final HttpServletRequest req;
-  
-  public HttpRequestContentStream( HttpServletRequest req ) {
-    this.req = req;
-    
-    contentType = req.getContentType();
-    // name = ???
-    // sourceInfo = ???
+  /**
+   * Wrap an HttpServletRequest as a ContentStream
+   */
+  static class HttpRequestContentStream extends ContentStreamBase
+  {
+    private final HttpServletRequest req;
     
-    String v = req.getHeader( "Content-Length" );
-    if( v != null ) {
-      size = Long.valueOf( v );
+    public HttpRequestContentStream( HttpServletRequest req ) {
+      this.req = req;
+      
+      contentType = req.getContentType();
+      // name = ???
+      // sourceInfo = ???
+      
+      String v = req.getHeader( "Content-Length" );
+      if( v != null ) {
+        size = Long.valueOf( v );
+      }
     }
-  }
 
-  @Override
-  public InputStream getStream() throws IOException {
-    return req.getInputStream();
+    @Override
+    public InputStream getStream() throws IOException {
+      return req.getInputStream();
+    }
   }
-}
 
 
-/**
- * Wrap a FileItem as a ContentStream
- */
-class FileItemContentStream extends ContentStreamBase
-{
-  private final FileItem item;
-  
-  public FileItemContentStream( FileItem f )
+  /**
+   * Wrap a FileItem as a ContentStream
+   */
+  static class FileItemContentStream extends ContentStreamBase
   {
-    item = f;
-    contentType = item.getContentType();
-    name = item.getName();
-    sourceInfo = item.getFieldName();
-    size = item.getSize();
-  }
-    
-  @Override
-  public InputStream getStream() throws IOException {
-    return item.getInputStream();
-  }
-}
-
-/**
- * The raw parser just uses the params directly
- */
-class RawRequestParser implements SolrRequestParser
-{
-  @Override
-  public SolrParams parseParamsAndFillStreams( 
-      final HttpServletRequest req, ArrayList<ContentStream> streams ) throws Exception
+    private final FileItem item;
+    
+    public FileItemContentStream( FileItem f )
+    {
+      item = f;
+      contentType = item.getContentType();
+      name = item.getName();
+      sourceInfo = item.getFieldName();
+      size = item.getSize();
+    }
+      
+    @Override
+    public InputStream getStream() throws IOException {
+      return item.getInputStream();
+    }
+  }
+
+  /**
+   * The raw parser just uses the params directly
+   */
+  static class RawRequestParser implements SolrRequestParser
   {
-    streams.add( new HttpRequestContentStream( req ) );
-    return SolrRequestParsers.parseQueryString( req.getQueryString() );
+    @Override
+    public SolrParams parseParamsAndFillStreams( 
+        final HttpServletRequest req, ArrayList<ContentStream> streams ) throws Exception
+    {
+      streams.add( new HttpRequestContentStream( req ) );
+      return parseQueryString( req.getQueryString() );
+    }
   }
-}
 
 
 
-/**
- * Extract Multipart streams
- */
-class MultipartRequestParser implements SolrRequestParser
-{
-  private final int uploadLimitKB;
-  
-  public MultipartRequestParser( int limit )
-  {
-    uploadLimitKB = limit;
-  }
-  
-  @Override
-  public SolrParams parseParamsAndFillStreams( 
-      final HttpServletRequest req, ArrayList<ContentStream> streams ) throws Exception
+  /**
+   * Extract Multipart streams
+   */
+  static class MultipartRequestParser implements SolrRequestParser
   {
-    if( !ServletFileUpload.isMultipartContent(req) ) {
-      throw new SolrException( ErrorCode.BAD_REQUEST, "Not multipart content! "+req.getContentType() );
-    }
+    private final int uploadLimitKB;
     
-    MultiMapSolrParams params = SolrRequestParsers.parseQueryString( req.getQueryString() );
+    public MultipartRequestParser( int limit )
+    {
+      uploadLimitKB = limit;
+    }
     
-    // Create a factory for disk-based file items
-    DiskFileItemFactory factory = new DiskFileItemFactory();
+    @Override
+    public SolrParams parseParamsAndFillStreams( 
+        final HttpServletRequest req, ArrayList<ContentStream> streams ) throws Exception
+    {
+      if( !ServletFileUpload.isMultipartContent(req) ) {
+        throw new SolrException( ErrorCode.BAD_REQUEST, "Not multipart content! "+req.getContentType() );
+      }
+      
+      MultiMapSolrParams params = parseQueryString( req.getQueryString() );
+      
+      // Create a factory for disk-based file items
+      DiskFileItemFactory factory = new DiskFileItemFactory();
 
-    // Set factory constraints
-    // TODO - configure factory.setSizeThreshold(yourMaxMemorySize);
-    // TODO - configure factory.setRepository(yourTempDirectory);
-
-    // Create a new file upload handler
-    ServletFileUpload upload = new ServletFileUpload(factory);
-    upload.setSizeMax( ((long) uploadLimitKB) * 1024L );
-
-    // Parse the request
-    List items = upload.parseRequest(req);
-    Iterator iter = items.iterator();
-    while (iter.hasNext()) {
-        FileItem item = (FileItem) iter.next();
-
-        // If its a form field, put it in our parameter map
-        if (item.isFormField()) {
-          MultiMapSolrParams.addParam( 
-            item.getFieldName(), 
-            item.getString(), params.getMap() );
-        }
-        // Add the stream
-        else { 
-          streams.add( new FileItemContentStream( item ) );
-        }
+      // Set factory constraints
+      // TODO - configure factory.setSizeThreshold(yourMaxMemorySize);
+      // TODO - configure factory.setRepository(yourTempDirectory);
+
+      // Create a new file upload handler
+      ServletFileUpload upload = new ServletFileUpload(factory);
+      upload.setSizeMax( ((long) uploadLimitKB) * 1024L );
+
+      // Parse the request
+      List items = upload.parseRequest(req);
+      Iterator iter = items.iterator();
+      while (iter.hasNext()) {
+          FileItem item = (FileItem) iter.next();
+
+          // If its a form field, put it in our parameter map
+          if (item.isFormField()) {
+            MultiMapSolrParams.addParam( 
+              item.getFieldName(), 
+              item.getString(), params.getMap() );
+          }
+          // Add the stream
+          else { 
+            streams.add( new FileItemContentStream( item ) );
+          }
+      }
+      return params;
     }
-    return params;
   }
-}
 
 
-/**
- * Extract application/x-www-form-urlencoded form data for POST requests
- */
-class FormDataRequestParser implements SolrRequestParser
-{
-  private final int uploadLimitKB;
-  
-  public FormDataRequestParser( int limit )
-  {
-    uploadLimitKB = limit;
-  }
-  
-  @Override
-  public SolrParams parseParamsAndFillStreams( 
-      final HttpServletRequest req, ArrayList<ContentStream> streams ) throws Exception
+  /**
+   * Extract application/x-www-form-urlencoded form data for POST requests
+   */
+  static class FormDataRequestParser implements SolrRequestParser
   {
-    if (!isFormData(req)) {
-      throw new SolrException( ErrorCode.BAD_REQUEST, "Not application/x-www-form-urlencoded content: "+req.getContentType() );
-    }
-    
-    final Map<String,String[]> map = new HashMap<String, String[]>();
+    private final int uploadLimitKB;
     
-    // also add possible URL parameters and include into the map (parsed using UTF-8):
-    final String qs = req.getQueryString();
-    if (qs != null) {
-      SolrRequestParsers.parseQueryString(qs, map);
+    public FormDataRequestParser( int limit )
+    {
+      uploadLimitKB = limit;
     }
     
-    // may be -1, so we check again later. But if its already greater we can stop processing!
-    final long totalLength = req.getContentLength();
-    final long maxLength = ((long) uploadLimitKB) * 1024L;
-    if (totalLength > maxLength) {
-      throw new SolrException(ErrorCode.BAD_REQUEST, "application/x-www-form-urlencoded content length (" +
-        totalLength + " bytes) exceeds upload limit of " + uploadLimitKB + " KB");
+    @Override
+    public SolrParams parseParamsAndFillStreams( 
+        final HttpServletRequest req, ArrayList<ContentStream> streams ) throws Exception
+    {
+      if (!isFormData(req)) {
+        throw new SolrException( ErrorCode.BAD_REQUEST, "Not application/x-www-form-urlencoded content: "+req.getContentType() );
+      }
+      
+      final Map<String,String[]> map = new HashMap<String, String[]>();
+      
+      // also add possible URL parameters and include into the map (parsed using UTF-8):
+      final String qs = req.getQueryString();
+      if (qs != null) {
+        parseQueryString(qs, map);
+      }
+      
+      // may be -1, so we check again later. But if its already greater we can stop processing!
+      final long totalLength = req.getContentLength();
+      final long maxLength = ((long) uploadLimitKB) * 1024L;
+      if (totalLength > maxLength) {
+        throw new SolrException(ErrorCode.BAD_REQUEST, "application/x-www-form-urlencoded content length (" +
+          totalLength + " bytes) exceeds upload limit of " + uploadLimitKB + " KB");
+      }
+      
+      // get query String from request body, using the charset given in content-type:
+      final String cs = ContentStreamBase.getCharsetFromContentType(req.getContentType());
+      final Charset charset = (cs == null) ? IOUtils.CHARSET_UTF_8 : Charset.forName(cs);
+      InputStream in = null;
+      try {
+        in = req.getInputStream();
+        final long bytesRead = parseFormDataContent(FastInputStream.wrap(in), maxLength, charset, map, false);
+        if (bytesRead == 0L && totalLength > 0L) {
+          throw getParameterIncompatibilityException();
+        }
+      } catch (IOException ioe) {
+        throw new SolrException(ErrorCode.BAD_REQUEST, ioe);
+      } catch (IllegalStateException ise) {
+        throw (SolrException) getParameterIncompatibilityException().initCause(ise);
+      } finally {
+        IOUtils.closeWhileHandlingException(in);
+      }
+      
+      return new MultiMapSolrParams(map);
     }
     
-    // get query String from request body, using the charset given in content-type:
-    final String cs = ContentStreamBase.getCharsetFromContentType(req.getContentType());
-    final Charset charset = (cs == null) ? IOUtils.CHARSET_UTF_8 : Charset.forName(cs);
-    InputStream in = null;
-    try {
-      in = req.getInputStream();
-      final long bytesRead = SolrRequestParsers.parseFormDataContent(FastInputStream.wrap(in), maxLength, charset, map);
-      if (bytesRead == 0L && totalLength > 0L) {
-        throw getParameterIncompatibilityException();
-      }
-    } catch (IOException ioe) {
-      throw new SolrException(ErrorCode.BAD_REQUEST, ioe);
-    } catch (IllegalStateException ise) {
-      throw (SolrException) getParameterIncompatibilityException().initCause(ise);
-    } finally {
-      IOUtils.closeWhileHandlingException(in);
+    private SolrException getParameterIncompatibilityException() {
+      return new SolrException(ErrorCode.SERVER_ERROR,
+        "Solr requires that request parameters sent using application/x-www-form-urlencoded " +
+        "content-type can be read through the request input stream. Unfortunately, the " +
+        "stream was empty / not available. This may be caused by another servlet filter calling " +
+        "ServletRequest.getParameter*() before SolrDispatchFilter, please remove it."
+      );
     }
     
-    return new MultiMapSolrParams(map);
-  }
-  
-  private SolrException getParameterIncompatibilityException() {
-    return new SolrException(ErrorCode.SERVER_ERROR,
-      "Solr requires that request parameters sent using application/x-www-form-urlencoded " +
-      "content-type can be read through the request input stream. Unfortunately, the " +
-      "stream was empty / not available. This may be caused by another servlet filter calling " +
-      "ServletRequest.getParameter*() before SolrDispatchFilter, please remove it."
-    );
-  }
-  
-  public boolean isFormData(HttpServletRequest req) {
-    String contentType = req.getContentType();
-    if (contentType != null) {
-      int idx = contentType.indexOf( ';' );
-      if( idx > 0 ) { // remove the charset definition "; charset=utf-8"
-        contentType = contentType.substring( 0, idx );
-      }
-      contentType = contentType.trim();
-      if( "application/x-www-form-urlencoded".equalsIgnoreCase(contentType)) {
-        return true;
+    public boolean isFormData(HttpServletRequest req) {
+      String contentType = req.getContentType();
+      if (contentType != null) {
+        int idx = contentType.indexOf( ';' );
+        if( idx > 0 ) { // remove the charset definition "; charset=utf-8"
+          contentType = contentType.substring( 0, idx );
+        }
+        contentType = contentType.trim();
+        if( "application/x-www-form-urlencoded".equalsIgnoreCase(contentType)) {
+          return true;
+        }
       }
+      return false;
     }
-    return false;
   }
-}
 
 
-/**
- * The default Logic
- */
-class StandardRequestParser implements SolrRequestParser
-{
-  MultipartRequestParser multipart;
-  RawRequestParser raw;
-  FormDataRequestParser formdata;
-  
-  StandardRequestParser(MultipartRequestParser multi, RawRequestParser raw, FormDataRequestParser formdata) 
-  {
-    this.multipart = multi;
-    this.raw = raw;
-    this.formdata = formdata;
-  }
-  
-  @Override
-  public SolrParams parseParamsAndFillStreams( 
-      final HttpServletRequest req, ArrayList<ContentStream> streams ) throws Exception
+  /**
+   * The default Logic
+   */
+  static class StandardRequestParser implements SolrRequestParser
   {
-    String method = req.getMethod().toUpperCase(Locale.ROOT);
-    if ("GET".equals(method) || "HEAD".equals(method) 
-        || ("PUT".equals(method) && req.getRequestURI().contains("/schema"))) {
-      return SolrRequestParsers.parseQueryString(req.getQueryString());
-    }
-    if ("POST".equals( method ) ) {
-      if (formdata.isFormData(req)) {
-        return formdata.parseParamsAndFillStreams(req, streams);
-      }
-      if (ServletFileUpload.isMultipartContent(req)) {
-        return multipart.parseParamsAndFillStreams(req, streams);
+    MultipartRequestParser multipart;
+    RawRequestParser raw;
+    FormDataRequestParser formdata;
+    
+    StandardRequestParser(MultipartRequestParser multi, RawRequestParser raw, FormDataRequestParser formdata) 
+    {
+      this.multipart = multi;
+      this.raw = raw;
+      this.formdata = formdata;
+    }
+    
+    @Override
+    public SolrParams parseParamsAndFillStreams( 
+        final HttpServletRequest req, ArrayList<ContentStream> streams ) throws Exception
+    {
+      String method = req.getMethod().toUpperCase(Locale.ROOT);
+      if ("GET".equals(method) || "HEAD".equals(method) 
+          || ("PUT".equals(method) && req.getRequestURI().contains("/schema"))) {
+        return parseQueryString(req.getQueryString());
+      }
+      if ("POST".equals( method ) ) {
+        if (formdata.isFormData(req)) {
+          return formdata.parseParamsAndFillStreams(req, streams);
+        }
+        if (ServletFileUpload.isMultipartContent(req)) {
+          return multipart.parseParamsAndFillStreams(req, streams);
+        }
+        return raw.parseParamsAndFillStreams(req, streams);
       }
-      return raw.parseParamsAndFillStreams(req, streams);
+      throw new SolrException(ErrorCode.BAD_REQUEST, "Unsupported method: " + method + " for request " + req);
     }
-    throw new SolrException(ErrorCode.BAD_REQUEST, "Unsupported method: " + method + " for request " + req);
   }
-}
-
-
-
-
-
-
-
-
-
+}
\ No newline at end of file

Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/spelling/SpellCheckCollator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/spelling/SpellCheckCollator.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/spelling/SpellCheckCollator.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/spelling/SpellCheckCollator.java Sun Aug 11 12:19:13 2013
@@ -202,7 +202,7 @@ public class SpellCheckCollator {
       //then be sure all of the new words have the same optional/required/prohibited status in the query.
       while(indexOfSpace>-1 && indexOfSpace<corr.length()-1) {
         addParenthesis = true;
-        char previousChar = tok.startOffset()>0 ? collation.charAt(tok.startOffset()-1) : ' ';
+        char previousChar = tok.startOffset()>0 ? origQuery.charAt(tok.startOffset()-1) : ' ';
         if(previousChar=='-' || previousChar=='+') {
           corrSb.insert(indexOfSpace + bump, previousChar);
           if(requiredOrProhibited==null) {

Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/spelling/SpellingQueryConverter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/spelling/SpellingQueryConverter.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/spelling/SpellingQueryConverter.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/spelling/SpellingQueryConverter.java Sun Aug 11 12:19:13 2013
@@ -18,8 +18,6 @@
 package org.apache.solr.spelling;
 
 import java.io.IOException;
-import java.io.Reader;
-import java.io.StringReader;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -28,11 +26,10 @@ import java.util.regex.Pattern;
 
 import org.apache.lucene.analysis.Token;
 import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
 import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
 
 
@@ -160,7 +157,7 @@ public class SpellingQueryConverter exte
         flagValue = TERM_PRECEDES_NEW_BOOLEAN_OPERATOR_FLAG;
       }
       try {
-        analyze(result, new StringReader(word), startIndex, flagValue);
+        analyze(result, word, startIndex, flagValue);
       } catch (IOException e) {
         // TODO: shouldn't we log something?
       }   
@@ -174,7 +171,7 @@ public class SpellingQueryConverter exte
     return result;
   }
   
-  protected void analyze(Collection<Token> result, Reader text, int offset, int flagsAttValue) throws IOException {
+  protected void analyze(Collection<Token> result, String text, int offset, int flagsAttValue) throws IOException {
     TokenStream stream = analyzer.tokenStream("", text);
     // TODO: support custom attributes
     CharTermAttribute termAtt = stream.addAttribute(CharTermAttribute.class);

Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/spelling/SuggestQueryConverter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/spelling/SuggestQueryConverter.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/spelling/SuggestQueryConverter.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/spelling/SuggestQueryConverter.java Sun Aug 11 12:19:13 2013
@@ -18,7 +18,6 @@ package org.apache.solr.spelling;
  */
 
 import java.io.IOException;
-import java.io.StringReader;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -38,7 +37,7 @@ public class SuggestQueryConverter exten
 
     Collection<Token> result = new ArrayList<Token>();
     try {
-      analyze(result, new StringReader(original), 0, 0);
+      analyze(result, original, 0, 0);
     } catch (IOException e) {
       throw new RuntimeException(e);
     }

Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/spelling/suggest/fst/FuzzyLookupFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/spelling/suggest/fst/FuzzyLookupFactory.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/spelling/suggest/fst/FuzzyLookupFactory.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/spelling/suggest/fst/FuzzyLookupFactory.java Sun Aug 11 12:19:13 2013
@@ -33,23 +33,33 @@ import org.apache.solr.spelling.suggest.
 public class FuzzyLookupFactory extends LookupFactory {
 
   /**
+   * If <code>true</code>, maxEdits, minFuzzyLength, transpositions and nonFuzzyPrefix 
+   * will be measured in Unicode code points (actual letters) instead of bytes.
+   */
+  public static final String UNICODE_AWARE = "unicodeAware";
+
+  /**
    * Maximum number of edits allowed, used by {@link LevenshteinAutomata#toAutomaton(int)}
+   * in bytes or Unicode code points (if {@link #UNICODE_AWARE} option is set to true).
    */
   public static final String MAX_EDITS = "maxEdits";
   
   /**
    * If transpositions are allowed, Fuzzy suggestions will be computed based on a primitive 
    * edit operation. If it is false, it will be based on the classic Levenshtein algorithm.
+   * Transpositions of bytes or Unicode code points (if {@link #UNICODE_AWARE} option is set to true).
    */
   public static final String TRANSPOSITIONS = "transpositions";
   
   /**
    * Length of common (non-fuzzy) prefix for the suggestions
+   * in bytes or Unicode code points (if {@link #UNICODE_AWARE} option is set to true).
    */
   public static final String NON_FUZZY_PREFIX = "nonFuzzyPrefix";
   
   /**
    * Minimum length of lookup key before any edits are allowed for the suggestions
+   * in bytes or Unicode code points (if {@link #UNICODE_AWARE} option is set to true).
    */
   public static final String MIN_FUZZY_LENGTH = "minFuzzyLength";
   
@@ -113,9 +123,13 @@ public class FuzzyLookupFactory extends 
     ? Integer.parseInt(params.get(MIN_FUZZY_LENGTH).toString())
     :FuzzySuggester.DEFAULT_MIN_FUZZY_LENGTH;
     
+    boolean unicodeAware = (params.get(UNICODE_AWARE) != null)
+    ? Boolean.valueOf(params.get(UNICODE_AWARE).toString())
+    : FuzzySuggester.DEFAULT_UNICODE_AWARE;
+    
     return new FuzzySuggester(indexAnalyzer, queryAnalyzer, options, 
         maxSurfaceFormsPerAnalyzedForm, maxGraphExpansions, maxEdits, 
-        transpositions, nonFuzzyPrefix, minFuzzyLength);
+        transpositions, nonFuzzyPrefix, minFuzzyLength, unicodeAware);
   }
 
   @Override

Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java Sun Aug 11 12:19:13 2013
@@ -187,6 +187,76 @@ public final class DefaultSolrCoreState 
       }
     }
   }
+  
+  @Override
+  public synchronized void closeIndexWriter(SolrCore core, boolean rollback)
+      throws IOException {
+    log.info("Closing IndexWriter...");
+    String coreName = core.getName();
+    synchronized (writerPauseLock) {
+      if (closed) {
+        throw new SolrException(ErrorCode.SERVICE_UNAVAILABLE, "Already closed");
+      }
+      
+      // we need to wait for the Writer to fall out of use
+      // first lets stop it from being lent out
+      pauseWriter = true;
+      // then lets wait until its out of use
+      log.info("Waiting until IndexWriter is unused... core=" + coreName);
+      
+      while (!writerFree) {
+        try {
+          writerPauseLock.wait(100);
+        } catch (InterruptedException e) {}
+        
+        if (closed) {
+          throw new SolrException(ErrorCode.SERVICE_UNAVAILABLE,
+              "SolrCoreState already closed");
+        }
+      }
+      
+      if (indexWriter != null) {
+        if (!rollback) {
+          try {
+            log.info("Closing old IndexWriter... core=" + coreName);
+            indexWriter.close();
+          } catch (Throwable t) {
+            SolrException.log(log, "Error closing old IndexWriter. core="
+                + coreName, t);
+          }
+        } else {
+          try {
+            log.info("Rollback old IndexWriter... core=" + coreName);
+            indexWriter.rollback();
+          } catch (Throwable t) {
+            SolrException.log(log, "Error rolling back old IndexWriter. core="
+                + coreName, t);
+          }
+        }
+      }
+      
+    }
+  }
+  
+  @Override
+  public synchronized void openIndexWriter(SolrCore core) throws IOException {
+    log.info("Creating new IndexWriter...");
+    synchronized (writerPauseLock) {
+      if (closed) {
+        throw new SolrException(ErrorCode.SERVICE_UNAVAILABLE, "Already closed");
+      }
+      
+      try {
+        indexWriter = createMainIndexWriter(core, "DirectUpdateHandler2");
+        log.info("New IndexWriter is ready to be used.");
+        // we need to null this so it picks up the new writer next get call
+        refCntWriter = null;
+      } finally {
+        pauseWriter = false;
+        writerPauseLock.notifyAll();
+      }
+    }
+  }
 
   @Override
   public synchronized void rollbackIndexWriter(SolrCore core) throws IOException {

Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java Sun Aug 11 12:19:13 2013
@@ -31,6 +31,7 @@ import java.util.concurrent.Future;
 import java.util.concurrent.atomic.AtomicLong;
 
 import org.apache.lucene.document.Document;
+import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.Term;
@@ -108,6 +109,8 @@ public class DirectUpdateHandler2 extend
     softCommitTracker = new CommitTracker("Soft", core, softCommitDocsUpperBound, softCommitTimeUpperBound, true, true);
     
     commitWithinSoftCommit = updateHandlerInfo.commitWithinSoftCommit;
+
+
   }
   
   public DirectUpdateHandler2(SolrCore core, UpdateHandler updateHandler) {
@@ -125,6 +128,13 @@ public class DirectUpdateHandler2 extend
     softCommitTracker = new CommitTracker("Soft", core, softCommitDocsUpperBound, softCommitTimeUpperBound, updateHandlerInfo.openSearcher, true);
     
     commitWithinSoftCommit = updateHandlerInfo.commitWithinSoftCommit;
+
+    UpdateLog existingLog = updateHandler.getUpdateLog();
+    if (this.ulog != null && this.ulog == existingLog) {
+      // If we are reusing the existing update log, inform the log that it's update handler has changed.
+      // We do this as late as possible.
+      this.ulog.init(this, core);
+    }
   }
 
   private void deleteAll() throws IOException {
@@ -423,11 +433,11 @@ public class DirectUpdateHandler2 extend
 
     log.info("start " + cmd);
     
-    IndexReader[] readers = cmd.readers;
-    if (readers != null && readers.length > 0) {
+    List<DirectoryReader> readers = cmd.readers;
+    if (readers != null && readers.size() > 0) {
       RefCounted<IndexWriter> iw = solrCoreState.getIndexWriter(core);
       try {
-        iw.get().addIndexes(readers);
+        iw.get().addIndexes(readers.toArray(new IndexReader[readers.size()]));
       } finally {
         iw.decref();
       }
@@ -531,11 +541,17 @@ public class DirectUpdateHandler2 extend
           }
           
           // SolrCore.verbose("writer.commit() start writer=",writer);
-          final Map<String,String> commitData = new HashMap<String,String>();
-          commitData.put(SolrIndexWriter.COMMIT_TIME_MSEC_KEY,
-              String.valueOf(System.currentTimeMillis()));
-          writer.setCommitData(commitData);
-          writer.commit();
+
+          if (writer.hasUncommittedChanges()) {
+            final Map<String,String> commitData = new HashMap<String,String>();
+            commitData.put(SolrIndexWriter.COMMIT_TIME_MSEC_KEY,
+                String.valueOf(System.currentTimeMillis()));
+            writer.setCommitData(commitData);
+            writer.commit();
+          } else {
+            log.info("No uncommitted changes. Skipping IW.commit.");
+          }
+
           // SolrCore.verbose("writer.commit() end");
           numDocsPending.set(0);
           callPostCommitCallbacks();

Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/MergeIndexesCommand.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/MergeIndexesCommand.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/MergeIndexesCommand.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/MergeIndexesCommand.java Sun Aug 11 12:19:13 2013
@@ -17,9 +17,14 @@
 
 package org.apache.solr.update;
 
+import com.google.common.base.Function;
+import com.google.common.base.Joiner;
+import com.google.common.collect.Iterables;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.solr.request.SolrQueryRequest;
 
+import java.util.List;
+
 /**
  * A merge indexes command encapsulated in an object.
  *
@@ -27,9 +32,9 @@ import org.apache.solr.request.SolrQuery
  *
  */
 public class MergeIndexesCommand extends UpdateCommand {
-  public DirectoryReader[] readers;
+  public List<DirectoryReader> readers;
 
-  public MergeIndexesCommand(DirectoryReader[] readers, SolrQueryRequest req) {
+  public MergeIndexesCommand(List<DirectoryReader> readers, SolrQueryRequest req) {
     super(req);
     this.readers = readers;
   }
@@ -42,12 +47,13 @@ public class MergeIndexesCommand extends
   @Override
   public String toString() {
     StringBuilder sb = new StringBuilder(super.toString());
-    if (readers != null && readers.length > 0) {
-      sb.append(readers[0].directory());
-      for (int i = 1; i < readers.length; i++) {
-        sb.append(",").append(readers[i].directory());
+    Joiner joiner = Joiner.on(",");
+    Iterable<String> directories = Iterables.transform(readers, new Function<DirectoryReader, String>() {
+      public String apply(DirectoryReader reader) {
+        return reader.directory().toString();
       }
-    }
+    });
+    joiner.skipNulls().join(sb, directories);
     sb.append('}');
     return sb.toString();
   }

Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java Sun Aug 11 12:19:13 2013
@@ -114,11 +114,11 @@ public class SolrCmdDistributor {
     
     // make sure any pending deletes are flushed
     flushDeletes(1);
-    
+
     // TODO: this is brittle
     // need to make a clone since these commands may be reused
     AddUpdateCommand clone = new AddUpdateCommand(null);
-    
+
     clone.solrDoc = cmd.solrDoc;
     clone.commitWithin = cmd.commitWithin;
     clone.overwrite = cmd.overwrite;
@@ -135,10 +135,79 @@ public class SolrCmdDistributor {
       }
       alist.add(addRequest);
     }
-    
+
     flushAdds(maxBufferedAddsPerServer);
   }
-  
+
+  /**
+   * Synchronous (blocking) add to specified node. Any error returned from node is propagated.
+   */
+  public void syncAdd(AddUpdateCommand cmd, Node node, ModifiableSolrParams params) throws IOException {
+    log.info("SYNCADD on {} : {}", node, cmd.getPrintableId());
+    checkResponses(false);
+    // flush all pending deletes
+    flushDeletes(1);
+    // flush all pending adds
+    flushAdds(1);
+    // finish with the pending requests
+    checkResponses(false);
+
+    UpdateRequestExt ureq = new UpdateRequestExt();
+    ureq.add(cmd.solrDoc, cmd.commitWithin, cmd.overwrite);
+    ureq.setParams(params);
+    syncRequest(node, ureq);
+  }
+
+  public void syncDelete(DeleteUpdateCommand cmd, List<Node> nodes, ModifiableSolrParams params) throws IOException {
+    log.info("SYNCDELETE on {} : ", nodes, cmd);
+    checkResponses(false);
+    // flush all pending adds
+    flushAdds(1);
+    // flush all pending deletes
+    flushDeletes(1);
+    // finish pending requests
+    checkResponses(false);
+
+    DeleteUpdateCommand clonedCmd = clone(cmd);
+    DeleteRequest deleteRequest = new DeleteRequest();
+    deleteRequest.cmd = clonedCmd;
+    deleteRequest.params = params;
+
+    UpdateRequestExt ureq = new UpdateRequestExt();
+    if (cmd.isDeleteById()) {
+      ureq.deleteById(cmd.getId(), cmd.getVersion());
+    } else {
+      ureq.deleteByQuery(cmd.query);
+    }
+    ureq.setParams(params);
+    for (Node node : nodes) {
+      syncRequest(node, ureq);
+    }
+  }
+
+  private void syncRequest(Node node, UpdateRequestExt ureq) {
+    Request sreq = new Request();
+    sreq.node = node;
+    sreq.ureq = ureq;
+
+    String url = node.getUrl();
+    String fullUrl;
+    if (!url.startsWith("http://") && !url.startsWith("https://")) {
+      fullUrl = "http://" + url;
+    } else {
+      fullUrl = url;
+    }
+
+    HttpSolrServer server = new HttpSolrServer(fullUrl,
+        updateShardHandler.getHttpClient());
+
+    try {
+      sreq.ursp = server.request(ureq);
+    } catch (Exception e) {
+      throw new SolrException(ErrorCode.SERVER_ERROR, "Failed synchronous update on shard " + sreq.node + " update: " + ureq , e);
+    }
+  }
+
   public void distribCommit(CommitUpdateCommand cmd, List<Node> nodes,
       ModifiableSolrParams params) throws IOException {
     

Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/SolrCoreState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/SolrCoreState.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/SolrCoreState.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/SolrCoreState.java Sun Aug 11 12:19:13 2013
@@ -88,6 +88,26 @@ public abstract class SolrCoreState {
    */
   public abstract void newIndexWriter(SolrCore core, boolean rollback) throws IOException;
   
+  
+  /**
+   * Expert method that closes the IndexWriter - you must call {@link #openIndexWriter(SolrCore)}
+   * in a finally block after calling this method.
+   * 
+   * @param core that the IW belongs to
+   * @param rollback true if IW should rollback rather than close
+   * @throws IOException If there is a low-level I/O error.
+   */
+  public abstract void closeIndexWriter(SolrCore core, boolean rollback) throws IOException;
+  
+  /**
+   * Expert method that opens the IndexWriter - you must call {@link #closeIndexWriter(SolrCore, boolean)}
+   * first, and then call this method in a finally block.
+   * 
+   * @param core that the IW belongs to
+   * @throws IOException If there is a low-level I/O error.
+   */
+  public abstract void openIndexWriter(SolrCore core) throws IOException;
+  
   /**
    * Get the current IndexWriter. If a new IndexWriter must be created, use the
    * settings from the given {@link SolrCore}.

Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java Sun Aug 11 12:19:13 2013
@@ -17,12 +17,15 @@
 
 package org.apache.solr.update;
 
+import org.apache.commons.io.FileUtils;
 import org.apache.lucene.index.*;
 import org.apache.lucene.index.IndexWriter.IndexReaderWarmer;
 import org.apache.lucene.util.InfoStream;
+import org.apache.lucene.util.PrintStreamInfoStream;
 import org.apache.lucene.util.Version;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.common.util.NamedList;
 import org.apache.solr.core.SolrConfig;
 import org.apache.solr.core.PluginInfo;
 import org.apache.solr.schema.IndexSchema;
@@ -30,6 +33,10 @@ import org.apache.solr.util.SolrPluginUt
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.PrintStream;
 import java.util.List;
 
 /**
@@ -43,7 +50,14 @@ public class SolrIndexConfig {
   public static final String DEFAULT_MERGE_SCHEDULER_CLASSNAME = ConcurrentMergeScheduler.class.getName();
   public final Version luceneVersion;
   
+  /**
+   * The explicit value of &lt;useCompoundFile&gt; specified on this index config
+   * @deprecated use {@link #getUseCompoundFile}
+   */
+  @Deprecated
   public final boolean useCompoundFile;
+  private boolean effectiveUseCompountFileSetting;
+
   public final int maxBufferedDocs;
   public final int maxMergeDocs;
   public final int maxIndexingThreads;
@@ -55,11 +69,10 @@ public class SolrIndexConfig {
   public final String lockType;
   public final PluginInfo mergePolicyInfo;
   public final PluginInfo mergeSchedulerInfo;
-  public final int termIndexInterval;
   
   public final PluginInfo mergedSegmentWarmerInfo;
   
-  public String infoStreamFile = null;
+  public InfoStream infoStream = InfoStream.NO_OUTPUT;
 
   // Available lock types
   public final static String LOCK_TYPE_SIMPLE = "simple";
@@ -73,7 +86,7 @@ public class SolrIndexConfig {
   @SuppressWarnings("deprecation")
   private SolrIndexConfig(SolrConfig solrConfig) {
     luceneVersion = solrConfig.luceneMatchVersion;
-    useCompoundFile = false;
+    useCompoundFile = effectiveUseCompountFileSetting = false;
     maxBufferedDocs = -1;
     maxMergeDocs = -1;
     maxIndexingThreads = IndexWriterConfig.DEFAULT_MAX_THREAD_STATES;
@@ -81,7 +94,6 @@ public class SolrIndexConfig {
     ramBufferSizeMB = 100;
     writeLockTimeout = -1;
     lockType = LOCK_TYPE_NATIVE;
-    termIndexInterval = IndexWriterConfig.DEFAULT_TERM_INDEX_INTERVAL;
     mergePolicyInfo = null;
     mergeSchedulerInfo = null;
     defaultMergePolicyClassName = TieredMergePolicy.class.getName();
@@ -105,15 +117,19 @@ public class SolrIndexConfig {
       def = new SolrIndexConfig(solrConfig);
     }
 
+    // sanity check: this will throw an error for us if there is more then one
+    // config section
+    Object unused = solrConfig.getNode(prefix, false);
+
     luceneVersion = solrConfig.luceneMatchVersion;
 
     // Assert that end-of-life parameters or syntax is not in our config.
     // Warn for luceneMatchVersion's before LUCENE_36, fail fast above
     assertWarnOrFail("The <mergeScheduler>myclass</mergeScheduler> syntax is no longer supported in solrconfig.xml. Please use syntax <mergeScheduler class=\"myclass\"/> instead.",
-        !((solrConfig.get(prefix+"/mergeScheduler/text()",null) != null) && (solrConfig.get(prefix+"/mergeScheduler/@class",null) == null)),
+        !((solrConfig.getNode(prefix+"/mergeScheduler",false) != null) && (solrConfig.get(prefix+"/mergeScheduler/@class",null) == null)),
         true);
     assertWarnOrFail("The <mergePolicy>myclass</mergePolicy> syntax is no longer supported in solrconfig.xml. Please use syntax <mergePolicy class=\"myclass\"/> instead.",
-        !((solrConfig.get(prefix+"/mergePolicy/text()",null) != null) && (solrConfig.get(prefix+"/mergePolicy/@class",null) == null)),
+        !((solrConfig.getNode(prefix+"/mergePolicy",false) != null) && (solrConfig.get(prefix+"/mergePolicy/@class",null) == null)),
         true);
     assertWarnOrFail("The <luceneAutoCommit>true|false</luceneAutoCommit> parameter is no longer valid in solrconfig.xml.",
         solrConfig.get(prefix+"/luceneAutoCommit", null) == null,
@@ -121,6 +137,7 @@ public class SolrIndexConfig {
 
     defaultMergePolicyClassName = def.defaultMergePolicyClassName;
     useCompoundFile=solrConfig.getBool(prefix+"/useCompoundFile", def.useCompoundFile);
+    effectiveUseCompountFileSetting = useCompoundFile;
     maxBufferedDocs=solrConfig.getInt(prefix+"/maxBufferedDocs",def.maxBufferedDocs);
     maxMergeDocs=solrConfig.getInt(prefix+"/maxMergeDocs",def.maxMergeDocs);
     maxIndexingThreads=solrConfig.getInt(prefix+"/maxIndexingThreads",def.maxIndexingThreads);
@@ -133,14 +150,21 @@ public class SolrIndexConfig {
     mergeSchedulerInfo = getPluginInfo(prefix + "/mergeScheduler", solrConfig, def.mergeSchedulerInfo);
     mergePolicyInfo = getPluginInfo(prefix + "/mergePolicy", solrConfig, def.mergePolicyInfo);
     
-    termIndexInterval = solrConfig.getInt(prefix + "/termIndexInterval", def.termIndexInterval);
-    
+    String val = solrConfig.get(prefix + "/termIndexInterval", null);
+    if (val != null) {
+      throw new IllegalArgumentException("Illegal parameter 'termIndexInterval'");
+    }
+
     boolean infoStreamEnabled = solrConfig.getBool(prefix + "/infoStream", false);
     if(infoStreamEnabled) {
-      infoStreamFile= solrConfig.get(prefix + "/infoStream/@file", null);
-      log.info("IndexWriter infoStream debug log is enabled: " + infoStreamFile);
+      String infoStreamFile = solrConfig.get(prefix + "/infoStream/@file", null);
+      if (infoStreamFile == null) {
+        log.info("IndexWriter infoStream solr logging is enabled");
+        infoStream = new LoggingInfoStream();
+      } else {
+        throw new IllegalArgumentException("Remove @file from <infoStream> to output messages to solr's logfile");
+      }
     }
-    
     mergedSegmentWarmerInfo = getPluginInfo(prefix + "/mergedSegmentWarmer", solrConfig, def.mergedSegmentWarmerInfo);
     if (mergedSegmentWarmerInfo != null && solrConfig.reopenReaders == false) {
       throw new IllegalArgumentException("Supplying a mergedSegmentWarmer will do nothing since reopenReaders is false");
@@ -179,15 +203,17 @@ public class SolrIndexConfig {
     if (ramBufferSizeMB != -1)
       iwc.setRAMBufferSizeMB(ramBufferSizeMB);
 
-    if (termIndexInterval != -1)
-      iwc.setTermIndexInterval(termIndexInterval);
-
     if (writeLockTimeout != -1)
       iwc.setWriteLockTimeout(writeLockTimeout);
 
     iwc.setSimilarity(schema.getSimilarity());
     iwc.setMergePolicy(buildMergePolicy(schema));
     iwc.setMergeScheduler(buildMergeScheduler(schema));
+    iwc.setInfoStream(infoStream);
+
+    // do this after buildMergePolicy since the backcompat logic 
+    // there may modify the effective useCompoundFile
+    iwc.setUseCompoundFile(getUseCompoundFile());
 
     if (maxIndexingThreads != -1) {
       iwc.setMaxThreadStates(maxIndexingThreads);
@@ -199,13 +225,22 @@ public class SolrIndexConfig {
                                                                         IndexReaderWarmer.class,
                                                                         null,
                                                                         new Class[] { InfoStream.class },
-                                                                        new Object[] { InfoStream.NO_OUTPUT });
+                                                                        new Object[] { iwc.getInfoStream() });
       iwc.setMergedSegmentWarmer(warmer);
     }
 
     return iwc;
   }
 
+  /**
+   * Builds a MergePolicy, may also modify the value returned by
+   * getUseCompoundFile() for use by the IndexWriterConfig if 
+   * "useCompoundFile" is specified as an init arg for 
+   * an out of the box MergePolicy that no longer supports it
+   *
+   * @see #fixUseCFMergePolicyInitArg
+   * @see #getUseCompoundFile
+   */
   private MergePolicy buildMergePolicy(IndexSchema schema) {
     String mpClassName = mergePolicyInfo == null ? defaultMergePolicyClassName : mergePolicyInfo.className;
 
@@ -213,25 +248,31 @@ public class SolrIndexConfig {
 
     if (policy instanceof LogMergePolicy) {
       LogMergePolicy logMergePolicy = (LogMergePolicy) policy;
+      fixUseCFMergePolicyInitArg(LogMergePolicy.class);
 
       if (maxMergeDocs != -1)
         logMergePolicy.setMaxMergeDocs(maxMergeDocs);
 
-      logMergePolicy.setUseCompoundFile(useCompoundFile);
+      logMergePolicy.setNoCFSRatio(getUseCompoundFile() ? 1.0 : 0.0);
 
       if (mergeFactor != -1)
         logMergePolicy.setMergeFactor(mergeFactor);
+
+
     } else if (policy instanceof TieredMergePolicy) {
       TieredMergePolicy tieredMergePolicy = (TieredMergePolicy) policy;
+      fixUseCFMergePolicyInitArg(TieredMergePolicy.class);
       
-      tieredMergePolicy.setUseCompoundFile(useCompoundFile);
+      tieredMergePolicy.setNoCFSRatio(getUseCompoundFile() ? 1.0 : 0.0);
       
       if (mergeFactor != -1) {
         tieredMergePolicy.setMaxMergeAtOnce(mergeFactor);
         tieredMergePolicy.setSegmentsPerTier(mergeFactor);
       }
-    } else {
-      log.warn("Use of compound file format or mergefactor cannot be configured if merge policy is not an instance of LogMergePolicy or TieredMergePolicy. The configured policy's defaults will be used.");
+
+
+    } else if (mergeFactor != -1) {
+      log.warn("Use of <mergeFactor> cannot be configured if merge policy is not an instance of LogMergePolicy or TieredMergePolicy. The configured policy's defaults will be used.");
     }
 
     if (mergePolicyInfo != null)
@@ -244,9 +285,58 @@ public class SolrIndexConfig {
     String msClassName = mergeSchedulerInfo == null ? SolrIndexConfig.DEFAULT_MERGE_SCHEDULER_CLASSNAME : mergeSchedulerInfo.className;
     MergeScheduler scheduler = schema.getResourceLoader().newInstance(msClassName, MergeScheduler.class);
 
-    if (mergeSchedulerInfo != null)
-      SolrPluginUtils.invokeSetters(scheduler, mergeSchedulerInfo.initArgs);
+    if (mergeSchedulerInfo != null) {
+      // LUCENE-5080: these two setters are removed, so we have to invoke setMaxMergesAndThreads
+      // if someone has them configured.
+      if (scheduler instanceof ConcurrentMergeScheduler) {
+        NamedList args = mergeSchedulerInfo.initArgs.clone();
+        Integer maxMergeCount = (Integer) args.remove("maxMergeCount");
+        if (maxMergeCount == null) {
+          maxMergeCount = ((ConcurrentMergeScheduler) scheduler).getMaxMergeCount();
+        }
+        Integer maxThreadCount = (Integer) args.remove("maxThreadCount");
+        if (maxThreadCount == null) {
+          maxThreadCount = ((ConcurrentMergeScheduler) scheduler).getMaxThreadCount();
+        }
+        ((ConcurrentMergeScheduler)scheduler).setMaxMergesAndThreads(maxMergeCount, maxThreadCount);
+        SolrPluginUtils.invokeSetters(scheduler, args);
+      } else {
+        SolrPluginUtils.invokeSetters(scheduler, mergeSchedulerInfo.initArgs);
+      }
+    }
 
     return scheduler;
   }
+
+  public boolean getUseCompoundFile() {
+    return effectiveUseCompountFileSetting;
+  }
+
+  /**
+   * Lucene 4.4 removed the setUseCompoundFile(boolean) method from the two 
+   * conrete MergePolicies provided with Lucene/Solr and added it to the 
+   * IndexWRiterConfig.  
+   * In the event that users have a value explicitly configured for this 
+   * setter in their MergePolicy init args, we remove it from the MergePolicy 
+   * init args, update the 'effective' useCompoundFile setting used by the 
+   * IndexWriterConfig, and warn about discontinuing to use this init arg.
+   * 
+   * @see #getUseCompoundFile
+   */
+  private void fixUseCFMergePolicyInitArg(Class c) {
+
+    if (null == mergePolicyInfo || null == mergePolicyInfo.initArgs) return;
+
+    Object useCFSArg = mergePolicyInfo.initArgs.remove("useCompoundFile");
+    if (null != useCFSArg) {
+      log.warn("Ignoring 'useCompoundFile' specified as an init arg for the <mergePolicy> since it is no directly longer supported by " + c.getSimpleName());
+      if (useCFSArg instanceof Boolean) {
+        boolean cfs = ((Boolean)useCFSArg).booleanValue();
+        log.warn("Please update your config to specify <useCompoundFile>"+cfs+"</useCompoundFile> directly in your <indexConfig> settings.");
+        effectiveUseCompountFileSetting = cfs;
+      } else {
+        log.error("MergePolicy's 'useCompoundFile' init arg is not a boolean, can not apply back compat logic to apply to the IndexWriterConfig: " + useCFSArg.toString());
+      }
+    }
+  }
 }

Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java Sun Aug 11 12:19:13 2013
@@ -77,7 +77,7 @@ public class SolrIndexWriter extends Ind
     super(directory,
           config.toIndexWriterConfig(schema).
           setOpenMode(create ? IndexWriterConfig.OpenMode.CREATE : IndexWriterConfig.OpenMode.APPEND).
-          setIndexDeletionPolicy(delPolicy).setCodec(codec).setInfoStream(toInfoStream(config))
+          setIndexDeletionPolicy(delPolicy).setCodec(codec)
           );
     log.debug("Opened Writer " + name);
     this.name = name;
@@ -88,20 +88,6 @@ public class SolrIndexWriter extends Ind
     this.directoryFactory = factory;
   }
 
-  private static InfoStream toInfoStream(SolrIndexConfig config) throws IOException {
-    String infoStreamFile = config.infoStreamFile;
-    if (infoStreamFile != null) {
-      File f = new File(infoStreamFile);
-      File parent = f.getParentFile();
-      if (parent != null) parent.mkdirs();
-      FileOutputStream fos = new FileOutputStream(f, true);
-      return new PrintStreamInfoStream(new PrintStream(fos, true, "UTF-8"));
-    } else {
-      return InfoStream.NO_OUTPUT;
-    }
-  }
-
-
   /**
    * use DocumentBuilder now...
    * private final void addField(Document doc, String name, String val) {
@@ -164,11 +150,8 @@ public class SolrIndexWriter extends Ind
       if (infoStream != null) {
         infoStream.close();
       }
-      
       isClosed = true;
-      
       directoryFactory.release(directory);
-      
       numCloses.incrementAndGet();
     }
   }

Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/TransactionLog.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/TransactionLog.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/TransactionLog.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/TransactionLog.java Sun Aug 11 12:19:13 2013
@@ -17,15 +17,6 @@
 
 package org.apache.solr.update;
 
-import org.apache.lucene.util.BytesRef;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.common.util.FastInputStream;
-import org.apache.solr.common.util.FastOutputStream;
-import org.apache.solr.common.util.JavaBinCodec;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 import java.io.File;
 import java.io.IOException;
 import java.io.OutputStream;
@@ -34,15 +25,23 @@ import java.nio.ByteBuffer;
 import java.nio.channels.Channels;
 import java.nio.channels.FileChannel;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
-import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicInteger;
 
+import org.apache.lucene.util.BytesRef;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.util.DataInputInputStream;
+import org.apache.solr.common.util.FastInputStream;
+import org.apache.solr.common.util.FastOutputStream;
+import org.apache.solr.common.util.JavaBinCodec;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 /**
  *  Log Format: List{Operation, Version, ...}
  *  ADD, VERSION, DOC
@@ -75,7 +74,7 @@ public class TransactionLog {
   FastOutputStream fos;    // all accesses to this stream should be synchronized on "this" (The TransactionLog)
   int numRecords;
   
-  volatile boolean deleteOnClose = true;  // we can delete old tlogs since they are currently only used for real-time-get (and in the future, recovery)
+  protected volatile boolean deleteOnClose = true;  // we can delete old tlogs since they are currently only used for real-time-get (and in the future, recovery)
 
   AtomicInteger refcount = new AtomicInteger(1);
   Map<String,Integer> globalStringMap = new HashMap<String, Integer>();
@@ -98,7 +97,7 @@ public class TransactionLog {
   };
 
   public class LogCodec extends JavaBinCodec {
-    public LogCodec() {
+    public LogCodec(JavaBinCodec.ObjectResolver resolver) {
       super(resolver);
     }
 
@@ -121,7 +120,7 @@ public class TransactionLog {
     }
 
     @Override
-    public String readExternString(FastInputStream fis) throws IOException {
+    public String readExternString(DataInputInputStream fis) throws IOException {
       int idx = readSize(fis);
       if (idx != 0) {// idx != 0 is the index of the extern string
       // no need to synchronize globalStringList - it's only updated before the first record is written to the log
@@ -191,6 +190,9 @@ public class TransactionLog {
     }
   }
 
+  // for subclasses
+  protected TransactionLog() {}
+
   /** Returns the number of records in the log (currently includes the header and an optional commit).
    * Note: currently returns 0 for reopened existing log files.
    */
@@ -245,7 +247,7 @@ public class TransactionLog {
 
 
   public long writeData(Object o) {
-    LogCodec codec = new LogCodec();
+    LogCodec codec = new LogCodec(resolver);
     try {
       long pos = fos.size();   // if we had flushed, this should be equal to channel.position()
       codec.init(fos);
@@ -260,7 +262,7 @@ public class TransactionLog {
   private void readHeader(FastInputStream fis) throws IOException {
     // read existing header
     fis = fis != null ? fis : new ChannelFastInputStream(channel, 0);
-    LogCodec codec = new LogCodec();
+    LogCodec codec = new LogCodec(resolver);
     Map header = (Map)codec.unmarshal(fis);
 
     fis.readInt(); // skip size
@@ -276,7 +278,7 @@ public class TransactionLog {
     }
   }
 
-  private void addGlobalStrings(Collection<String> strings) {
+  protected void addGlobalStrings(Collection<String> strings) {
     if (strings == null) return;
     int origSize = globalStringMap.size();
     for (String s : strings) {
@@ -297,7 +299,7 @@ public class TransactionLog {
     }
   }
 
-  private void writeLogHeader(LogCodec codec) throws IOException {
+  protected void writeLogHeader(LogCodec codec) throws IOException {
     long pos = fos.size();
     assert pos == 0;
 
@@ -309,7 +311,7 @@ public class TransactionLog {
     endRecord(pos);
   }
 
-  private void endRecord(long startRecordPosition) throws IOException {
+  protected void endRecord(long startRecordPosition) throws IOException {
     fos.writeInt((int)(fos.size() - startRecordPosition));
     numRecords++;
   }
@@ -333,7 +335,7 @@ public class TransactionLog {
   int lastAddSize;
 
   public long write(AddUpdateCommand cmd, int flags) {
-    LogCodec codec = new LogCodec();
+    LogCodec codec = new LogCodec(resolver);
     SolrInputDocument sdoc = cmd.getSolrInputDocument();
 
     try {
@@ -375,7 +377,7 @@ public class TransactionLog {
   }
 
   public long writeDelete(DeleteUpdateCommand cmd, int flags) {
-    LogCodec codec = new LogCodec();
+    LogCodec codec = new LogCodec(resolver);
 
     try {
       checkWriteHeader(codec, null);
@@ -405,7 +407,7 @@ public class TransactionLog {
   }
 
   public long writeDeleteByQuery(DeleteUpdateCommand cmd, int flags) {
-    LogCodec codec = new LogCodec();
+    LogCodec codec = new LogCodec(resolver);
     try {
       checkWriteHeader(codec, null);
 
@@ -431,7 +433,7 @@ public class TransactionLog {
 
 
   public long writeCommit(CommitUpdateCommand cmd, int flags) {
-    LogCodec codec = new LogCodec();
+    LogCodec codec = new LogCodec(resolver);
     synchronized (this) {
       try {
         long pos = fos.size();   // if we had flushed, this should be equal to channel.position()
@@ -479,7 +481,7 @@ public class TransactionLog {
       }
 
       ChannelFastInputStream fis = new ChannelFastInputStream(channel, pos);
-      LogCodec codec = new LogCodec();
+      LogCodec codec = new LogCodec(resolver);
       return codec.readVal(fis);
     } catch (IOException e) {
       throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
@@ -529,7 +531,7 @@ public class TransactionLog {
     }
   }
 
-  private void close() {
+  protected void close() {
     try {
       if (debug) {
         log.debug("Closing tlog" + this);
@@ -570,19 +572,22 @@ public class TransactionLog {
 
   /** Returns a single threaded reverse reader */
   public ReverseReader getReverseReader() throws IOException {
-    return new ReverseReader();
+    return new FSReverseReader();
   }
 
 
   public class LogReader {
-    ChannelFastInputStream fis;
-    private LogCodec codec = new LogCodec();
+    private ChannelFastInputStream fis;
+    private LogCodec codec = new LogCodec(resolver);
 
     public LogReader(long startingPos) {
       incref();
       fis = new ChannelFastInputStream(channel, startingPos);
     }
 
+    // for classes that extend
+    protected LogReader() {}
+
     /** Returns the next object from the log, or null if none available.
      *
      * @return The log record, or null if EOF
@@ -638,11 +643,32 @@ public class TransactionLog {
 
   }
 
-  public class ReverseReader {
+  public abstract class ReverseReader {
+
+
+
+    /** Returns the next object from the log, or null if none available.
+     *
+     * @return The log record, or null if EOF
+     * @throws IOException If there is a low-level I/O error.
+     */
+    public abstract Object next() throws IOException;
+
+    /* returns the position in the log file of the last record returned by next() */
+    public abstract long position();
+    public abstract void close();
+
+    @Override
+    public abstract String toString() ;
+
+
+  }
+  
+  public class FSReverseReader extends ReverseReader {
     ChannelFastInputStream fis;
-    private LogCodec codec = new LogCodec() {
+    private LogCodec codec = new LogCodec(resolver) {
       @Override
-      public SolrInputDocument readSolrInputDocument(FastInputStream dis) {
+      public SolrInputDocument readSolrInputDocument(DataInputInputStream dis) {
         // Given that the SolrInputDocument is last in an add record, it's OK to just skip
         // reading it completely.
         return null;
@@ -652,7 +678,7 @@ public class TransactionLog {
     int nextLength;  // length of the next record (the next one closer to the start of the log file)
     long prevPos;    // where we started reading from last time (so prevPos - nextLength == start of next record)
 
-    public ReverseReader() throws IOException {
+    public FSReverseReader() throws IOException {
       incref();
 
       long sz;

Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/UpdateHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/UpdateHandler.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/UpdateHandler.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/UpdateHandler.java Sun Aug 11 12:19:13 2013
@@ -18,10 +18,11 @@
 package org.apache.solr.update;
 
 
-import java.io.File;
 import java.io.IOException;
 import java.util.Vector;
 
+import org.apache.solr.core.DirectoryFactory;
+import org.apache.solr.core.HdfsDirectoryFactory;
 import org.apache.solr.core.PluginInfo;
 import org.apache.solr.core.SolrCore;
 import org.apache.solr.core.SolrEventListener;
@@ -52,7 +53,7 @@ public abstract class UpdateHandler impl
   protected Vector<SolrEventListener> softCommitCallbacks = new Vector<SolrEventListener>();
   protected Vector<SolrEventListener> optimizeCallbacks = new Vector<SolrEventListener>();
 
-  protected volatile UpdateLog ulog;
+  protected final UpdateLog ulog;
 
   private void parseEventListeners() {
     final Class<SolrEventListener> clazz = SolrEventListener.class;
@@ -71,34 +72,6 @@ public abstract class UpdateHandler impl
     }
   }
 
-
-  private void initLog(PluginInfo ulogPluginInfo) {
-    if (ulogPluginInfo != null && ulogPluginInfo.isEnabled()) {
-      ulog = new UpdateLog();
-      ulog.init(ulogPluginInfo);
-      // ulog = core.createInitInstance(ulogPluginInfo, UpdateLog.class, "update log", "solr.NullUpdateLog");
-      ulog.init(this, core);
-    }
-  }
-
-  // not thread safe - for startup
-  private void clearLog(PluginInfo ulogPluginInfo) {
-    if (ulogPluginInfo == null) return;
-    File tlogDir = UpdateLog.getTlogDir(core, ulogPluginInfo);
-    log.info("Clearing tlog files, tlogDir=" + tlogDir);
-    if (tlogDir.exists()) {
-      String[] files = UpdateLog.getLogList(tlogDir);
-      for (String file : files) {
-        File f = new File(tlogDir, file);
-        boolean s = f.delete();
-        if (!s) {
-          log.error("Could not remove tlog file:" + f.getAbsolutePath());
-          //throw new SolrException(ErrorCode.SERVER_ERROR, "Could not remove tlog file:" + f.getAbsolutePath());
-        }
-      }
-    }
-  }
-
   protected void callPostCommitCallbacks() {
     for (SolrEventListener listener : commitCallbacks) {
       listener.postCommit();
@@ -127,14 +100,43 @@ public abstract class UpdateHandler impl
     idFieldType = idField!=null ? idField.getType() : null;
     parseEventListeners();
     PluginInfo ulogPluginInfo = core.getSolrConfig().getPluginInfo(UpdateLog.class.getName());
-    if (!core.isReloaded() && !core.getDirectoryFactory().isPersistent()) {
-      clearLog(ulogPluginInfo);
-    }
-    if (updateLog == null) {
-      initLog(ulogPluginInfo);
+    
+
+    if (updateLog == null && ulogPluginInfo != null && ulogPluginInfo.isEnabled()) {
+      String dataDir = (String)ulogPluginInfo.initArgs.get("dir");
+      
+      String ulogDir = core.getCoreDescriptor().getUlogDir();
+      if (ulogDir != null) {
+        dataDir = ulogDir;
+      }
+      if (dataDir == null || dataDir.length()==0) {
+        dataDir = core.getDataDir();
+      }
+           
+      if (dataDir != null && dataDir.startsWith("hdfs:/")) {
+        DirectoryFactory dirFactory = core.getDirectoryFactory();
+        if (dirFactory instanceof HdfsDirectoryFactory) {
+          ulog = new HdfsUpdateLog(((HdfsDirectoryFactory)dirFactory).getConfDir());
+        } else {
+          ulog = new HdfsUpdateLog();
+        }
+        
+      } else {
+        ulog = new UpdateLog();
+      }
+      
+      if (!core.isReloaded() && !core.getDirectoryFactory().isPersistent()) {
+        ulog.clearLog(core, ulogPluginInfo);
+      }
+      
+      ulog.init(ulogPluginInfo);
+
+      ulog.init(this, core);
     } else {
-      this.ulog = updateLog;
+      ulog = updateLog;
     }
+    // ulog.init() when reusing an existing log is deferred (currently at the end of the DUH2 constructor
+
   }
 
   /**

Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/UpdateLog.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/UpdateLog.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/UpdateLog.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/UpdateLog.java Sun Aug 11 12:19:13 2013
@@ -17,15 +17,38 @@
 
 package org.apache.solr.update;
 
+import static org.apache.solr.update.processor.DistributedUpdateProcessor.DistribPhase.FROMLEADER;
+import static org.apache.solr.update.processor.DistributingUpdateProcessorFactory.DISTRIB_UPDATE_PARAM;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Deque;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.Locale;
+import java.util.Map;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.Future;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.fs.FileSystem;
 import org.apache.lucene.util.BytesRef;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.SolrException.ErrorCode;
 import org.apache.solr.common.SolrInputDocument;
 import org.apache.solr.common.params.ModifiableSolrParams;
 import org.apache.solr.common.params.SolrParams;
-import org.apache.solr.common.params.UpdateParams;
 import org.apache.solr.common.util.ExecutorUtil;
-import org.apache.solr.common.util.NamedList;
 import org.apache.solr.core.PluginInfo;
 import org.apache.solr.core.SolrCore;
 import org.apache.solr.request.LocalSolrQueryRequest;
@@ -34,9 +57,6 @@ import org.apache.solr.request.SolrReque
 import org.apache.solr.response.SolrQueryResponse;
 import org.apache.solr.search.SolrIndexSearcher;
 import org.apache.solr.update.processor.DistributedUpdateProcessor;
-import org.apache.solr.update.processor.DistributedUpdateProcessorFactory;
-import org.apache.solr.update.processor.DistributingUpdateProcessorFactory;
-import org.apache.solr.update.processor.RunUpdateProcessorFactory;
 import org.apache.solr.update.processor.UpdateRequestProcessor;
 import org.apache.solr.update.processor.UpdateRequestProcessorChain;
 import org.apache.solr.util.DefaultSolrThreadFactory;
@@ -45,15 +65,6 @@ import org.apache.solr.util.plugin.Plugi
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.File;
-import java.io.FilenameFilter;
-import java.io.IOException;
-import java.util.*;
-import java.util.concurrent.*;
-
-import static org.apache.solr.update.processor.DistributingUpdateProcessorFactory.DISTRIB_UPDATE_PARAM;
-import static org.apache.solr.update.processor.DistributedUpdateProcessor.DistribPhase.FROMLEADER;
-
 
 /** @lucene.experimental */
 public class UpdateLog implements PluginInfoInitialized {
@@ -64,6 +75,10 @@ public class UpdateLog implements Plugin
   public boolean debug = log.isDebugEnabled();
   public boolean trace = log.isTraceEnabled();
 
+  // TODO: hack
+  public FileSystem getFs() {
+    return null;
+  }
 
   public enum SyncLevel { NONE, FLUSH, FSYNC;
     public static SyncLevel getSyncLevel(String level){
@@ -108,27 +123,27 @@ public class UpdateLog implements Plugin
   }
 
   long id = -1;
-  private State state = State.ACTIVE;
-  private int operationFlags;  // flags to write in the transaction log with operations (i.e. FLAG_GAP)
+  protected State state = State.ACTIVE;
+  protected int operationFlags;  // flags to write in the transaction log with operations (i.e. FLAG_GAP)
 
-  private TransactionLog tlog;
-  private TransactionLog prevTlog;
-  private Deque<TransactionLog> logs = new LinkedList<TransactionLog>();  // list of recent logs, newest first
-  private LinkedList<TransactionLog> newestLogsOnStartup = new LinkedList<TransactionLog>();
-  private int numOldRecords;  // number of records in the recent logs
-
-  private Map<BytesRef,LogPtr> map = new HashMap<BytesRef, LogPtr>();
-  private Map<BytesRef,LogPtr> prevMap;  // used while committing/reopening is happening
-  private Map<BytesRef,LogPtr> prevMap2;  // used while committing/reopening is happening
-  private TransactionLog prevMapLog;  // the transaction log used to look up entries found in prevMap
-  private TransactionLog prevMapLog2;  // the transaction log used to look up entries found in prevMap
+  protected TransactionLog tlog;
+  protected TransactionLog prevTlog;
+  protected Deque<TransactionLog> logs = new LinkedList<TransactionLog>();  // list of recent logs, newest first
+  protected LinkedList<TransactionLog> newestLogsOnStartup = new LinkedList<TransactionLog>();
+  protected int numOldRecords;  // number of records in the recent logs
+
+  protected Map<BytesRef,LogPtr> map = new HashMap<BytesRef, LogPtr>();
+  protected Map<BytesRef,LogPtr> prevMap;  // used while committing/reopening is happening
+  protected Map<BytesRef,LogPtr> prevMap2;  // used while committing/reopening is happening
+  protected TransactionLog prevMapLog;  // the transaction log used to look up entries found in prevMap
+  protected TransactionLog prevMapLog2;  // the transaction log used to look up entries found in prevMap
 
-  private final int numDeletesToKeep = 1000;
-  private final int numDeletesByQueryToKeep = 100;
+  protected final int numDeletesToKeep = 1000;
+  protected final int numDeletesByQueryToKeep = 100;
   public final int numRecordsToKeep = 100;
 
   // keep track of deletes only... this is not updated on an add
-  private LinkedHashMap<BytesRef, LogPtr> oldDeletes = new LinkedHashMap<BytesRef, LogPtr>(numDeletesToKeep) {
+  protected LinkedHashMap<BytesRef, LogPtr> oldDeletes = new LinkedHashMap<BytesRef, LogPtr>(numDeletesToKeep) {
     @Override
     protected boolean removeEldestEntry(Map.Entry eldest) {
       return size() > numDeletesToKeep;
@@ -145,21 +160,21 @@ public class UpdateLog implements Plugin
     }
   }
 
-  private LinkedList<DBQ> deleteByQueries = new LinkedList<DBQ>();
+  protected LinkedList<DBQ> deleteByQueries = new LinkedList<DBQ>();
 
-  private String[] tlogFiles;
-  private File tlogDir;
-  private Collection<String> globalStrings;
+  protected String[] tlogFiles;
+  protected File tlogDir;
+  protected Collection<String> globalStrings;
 
-  private String dataDir;
-  private String lastDataDir;
+  protected String dataDir;
+  protected String lastDataDir;
 
-  private VersionInfo versionInfo;
+  protected VersionInfo versionInfo;
 
-  private SyncLevel defaultSyncLevel = SyncLevel.FLUSH;
+  protected SyncLevel defaultSyncLevel = SyncLevel.FLUSH;
 
   volatile UpdateHandler uhandler;    // a core reload can change this reference!
-  private volatile boolean cancelApplyBufferUpdate;
+  protected volatile boolean cancelApplyBufferUpdate;
   List<Long> startingVersions;
   int startingOperation;  // last operation in the logs on startup
 
@@ -189,13 +204,17 @@ public class UpdateLog implements Plugin
     defaultSyncLevel = SyncLevel.getSyncLevel((String)info.initArgs.get("syncLevel"));
   }
 
+  /* Note, when this is called, uhandler is not completely constructed.
+   * This must be called when a new log is created, or
+   * for an existing log whenever the core or update handler changes.
+   */
   public void init(UpdateHandler uhandler, SolrCore core) {
     // ulogDir from CoreDescriptor overrides
     String ulogDir = core.getCoreDescriptor().getUlogDir();
     if (ulogDir != null) {
       dataDir = ulogDir;
     }
-    
+
     if (dataDir == null || dataDir.length()==0) {
       dataDir = core.getDataDir();
     }
@@ -276,8 +295,8 @@ public class UpdateLog implements Plugin
 
   }
   
-  public File getLogDir() {
-    return tlogDir;
+  public String getLogDir() {
+    return tlogDir.getAbsolutePath();
   }
   
   public List<Long> getStartingVersions() {
@@ -291,7 +310,7 @@ public class UpdateLog implements Plugin
   /* Takes over ownership of the log, keeping it until no longer needed
      and then decrementing it's reference and dropping it.
    */
-  private void addOldLog(TransactionLog oldLog, boolean removeOld) {
+  protected void addOldLog(TransactionLog oldLog, boolean removeOld) {
     if (oldLog == null) return;
 
     numOldRecords += oldLog.numRecords();
@@ -322,7 +341,7 @@ public class UpdateLog implements Plugin
   }
 
 
-  public static String[] getLogList(File directory) {
+  public String[] getLogList(File directory) {
     final String prefix = TLOG_NAME+'.';
     String[] names = directory.list(new FilenameFilter() {
       @Override
@@ -330,6 +349,9 @@ public class UpdateLog implements Plugin
         return name.startsWith(prefix);
       }
     });
+    if (names == null) {
+      throw new RuntimeException(new FileNotFoundException(directory.getAbsolutePath()));
+    }
     Arrays.sort(names);
     return names;
   }
@@ -540,7 +562,7 @@ public class UpdateLog implements Plugin
     }
   }
 
-  private void newMap() {
+  protected void newMap() {
     prevMap2 = prevMap;
     prevMapLog2 = prevMapLog;
 
@@ -793,7 +815,7 @@ public class UpdateLog implements Plugin
   }
 
 
-  private void ensureLog() {
+  protected void ensureLog() {
     if (tlog == null) {
       String newLogName = String.format(Locale.ROOT, LOG_FILENAME_PATTERN, TLOG_NAME, id);
       tlog = new TransactionLog(new File(tlogDir, newLogName), globalStrings);
@@ -968,6 +990,8 @@ public class UpdateLog implements Plugin
               log.warn("Exception reverse reading log", ex);
               break;
             }
+
+            numUpdates++;
           }
 
         } catch (IOException e) {
@@ -1139,7 +1163,7 @@ public class UpdateLog implements Plugin
 
 
 
-  private RecoveryInfo recoveryInfo;
+  protected RecoveryInfo recoveryInfo;
 
   class LogReplayer implements Runnable {
     private Logger loglog = log;  // set to something different?
@@ -1416,7 +1440,7 @@ public class UpdateLog implements Plugin
     }
   }
   
-  public static File getTlogDir(SolrCore core, PluginInfo info) {
+  protected String getTlogDir(SolrCore core, PluginInfo info) {
     String dataDir = (String) info.initArgs.get("dir");
     
     String ulogDir = core.getCoreDescriptor().getUlogDir();
@@ -1427,11 +1451,30 @@ public class UpdateLog implements Plugin
     if (dataDir == null || dataDir.length() == 0) {
       dataDir = core.getDataDir();
     }
-    
-    return new File(dataDir, TLOG_NAME);
+
+    return dataDir + "/" + TLOG_NAME;
+  }
+  
+  /**
+   * Clears the logs on the file system. Only call before init.
+   * 
+   * @param core the SolrCore
+   * @param ulogPluginInfo the init info for the UpdateHandler
+   */
+  public void clearLog(SolrCore core, PluginInfo ulogPluginInfo) {
+    if (ulogPluginInfo == null) return;
+    File tlogDir = new File(getTlogDir(core, ulogPluginInfo));
+    if (tlogDir.exists()) {
+      String[] files = getLogList(tlogDir);
+      for (String file : files) {
+        File f = new File(tlogDir, file);
+        boolean s = f.delete();
+        if (!s) {
+          log.error("Could not remove tlog file:" + f);
+        }
+      }
+    }
   }
   
 }
 
-
-

Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/processor/CloneFieldUpdateProcessorFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/processor/CloneFieldUpdateProcessorFactory.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/processor/CloneFieldUpdateProcessorFactory.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/processor/CloneFieldUpdateProcessorFactory.java Sun Aug 11 12:19:13 2013
@@ -195,25 +195,13 @@ public class CloneFieldUpdateProcessorFa
     
     srcSelector = 
       FieldMutatingUpdateProcessor.createFieldNameSelector
-      (core.getResourceLoader(),
-       core,
-       srcInclusions.fieldName,
-       srcInclusions.typeName,
-       srcInclusions.typeClass,
-       srcInclusions.fieldRegex,
-       FieldMutatingUpdateProcessor.SELECT_NO_FIELDS);
+          (core.getResourceLoader(), core, srcInclusions, FieldMutatingUpdateProcessor.SELECT_NO_FIELDS);
 
     for (SelectorParams exc : srcExclusions) {
       srcSelector = FieldMutatingUpdateProcessor.wrap
         (srcSelector,
          FieldMutatingUpdateProcessor.createFieldNameSelector
-         (core.getResourceLoader(),
-          core,
-          exc.fieldName,
-          exc.typeName,
-          exc.typeClass,
-          exc.fieldRegex,
-          FieldMutatingUpdateProcessor.SELECT_NO_FIELDS));
+             (core.getResourceLoader(), core, exc, FieldMutatingUpdateProcessor.SELECT_NO_FIELDS));
     }
   }
 



Mime
View raw message