lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From no...@apache.org
Subject svn commit: r1644095 - in /lucene/dev/trunk/solr/core/src: java/org/apache/solr/core/ java/org/apache/solr/handler/ java/org/apache/solr/handler/admin/ java/org/apache/solr/request/ java/org/apache/solr/servlet/ java/org/apache/solr/util/ test/org/apac...
Date Tue, 09 Dec 2014 15:58:32 GMT
Author: noble
Date: Tue Dec  9 15:58:32 2014
New Revision: 1644095

URL: http://svn.apache.org/r1644095
Log:
SOLR-6787 API to manage blobs in Solr

Added:
    lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/BlobHandler.java   (with props)
    lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/TestBlobHandler.java   (with
props)
Modified:
    lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SolrCore.java
    lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
    lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java
    lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
    lucene/dev/trunk/solr/core/src/java/org/apache/solr/request/SolrQueryRequestBase.java
    lucene/dev/trunk/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java
    lucene/dev/trunk/solr/core/src/java/org/apache/solr/util/SimplePostTool.java
    lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/TestSolrConfigHandlerConcurrent.java
    lucene/dev/trunk/solr/core/src/test/org/apache/solr/servlet/SolrRequestParserTest.java
    lucene/dev/trunk/solr/core/src/test/org/apache/solr/util/SimplePostToolTest.java

Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SolrCore.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SolrCore.java?rev=1644095&r1=1644094&r2=1644095&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SolrCore.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SolrCore.java Tue Dec  9 15:58:32
2014
@@ -23,6 +23,7 @@ import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
+import java.io.OutputStream;
 import java.io.Writer;
 import java.lang.reflect.Constructor;
 import java.net.URL;
@@ -64,6 +65,7 @@ import org.apache.lucene.store.Directory
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.LockObtainFailedException;
+import org.apache.solr.client.solrj.impl.BinaryResponseParser;
 import org.apache.solr.cloud.CloudDescriptor;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.cloud.ClusterState;
@@ -75,6 +77,7 @@ import org.apache.solr.common.util.Execu
 import org.apache.solr.common.util.NamedList;
 import org.apache.solr.common.util.SimpleOrderedMap;
 import org.apache.solr.core.DirectoryFactory.DirContext;
+import org.apache.solr.handler.ReplicationHandler;
 import org.apache.solr.handler.RequestHandlerBase;
 import org.apache.solr.handler.SnapPuller;
 import org.apache.solr.handler.SolrConfigHandler;
@@ -2091,8 +2094,28 @@ public final class SolrCore implements S
     m.put("csv", new CSVResponseWriter());
     m.put("xsort", new SortingResponseWriter());
     m.put("schema.xml", new SchemaXmlResponseWriter());
+    m.put(ReplicationHandler.FILE_STREAM, getFileStreamWriter());
     DEFAULT_RESPONSE_WRITERS = Collections.unmodifiableMap(m);
   }
+
+  private static BinaryResponseWriter getFileStreamWriter() {
+    return new BinaryResponseWriter(){
+      @Override
+      public void write(OutputStream out, SolrQueryRequest req, SolrQueryResponse response)
throws IOException {
+        RawWriter rawWriter = (RawWriter) response.getValues().get(ReplicationHandler.FILE_STREAM);
+        rawWriter.write(out);
+      }
+
+      @Override
+      public String getContentType(SolrQueryRequest request, SolrQueryResponse response)
{
+        return BinaryResponseParser.BINARY_CONTENT_TYPE;
+      }
+    };
+  }
+
+  public interface RawWriter {
+    public void write(OutputStream os) throws IOException ;
+  }
   
   /** Configure the query response writers. There will always be a default writer; additional
    * writers may also be configured. */

Added: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/BlobHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/BlobHandler.java?rev=1644095&view=auto
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/BlobHandler.java (added)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/BlobHandler.java Tue Dec 
9 15:58:32 2014
@@ -0,0 +1,267 @@
+package org.apache.solr.handler;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.math.BigInteger;
+import java.nio.ByteBuffer;
+import java.security.MessageDigest;
+import java.text.MessageFormat;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.lucene.index.StorableField;
+import org.apache.lucene.index.StoredDocument;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.TopFieldDocs;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.cloud.ZkNodeProps;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.MapSolrParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.params.UpdateParams;
+import org.apache.solr.common.util.ContentStream;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.StrUtils;
+import org.apache.solr.core.PluginInfo;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.request.LocalSolrQueryRequest;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.request.SolrRequestHandler;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.schema.FieldType;
+import org.apache.solr.search.QParser;
+import org.apache.solr.update.AddUpdateCommand;
+import org.apache.solr.update.processor.UpdateRequestProcessor;
+import org.apache.solr.update.processor.UpdateRequestProcessorChain;
+import org.apache.solr.util.SimplePostTool;
+import org.apache.solr.util.plugin.PluginInfoInitialized;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.solr.common.cloud.ZkNodeProps.makeMap;
+
+public class BlobHandler extends RequestHandlerBase  implements PluginInfoInitialized{
+  protected static final Logger log = LoggerFactory.getLogger(BlobHandler.class);
+
+  private static final long MAX_SZ = 5*1024*1024;//2MB
+  private long maxSize = MAX_SZ;
+
+
+
+  @Override
+  public void handleRequestBody(final SolrQueryRequest req, SolrQueryResponse rsp) throws
Exception {
+    String httpMethod = (String) req.getContext().get("httpMethod");
+    String path = (String) req.getContext().get("path");
+    SolrConfigHandler.setWt(req,"json");
+
+    List<String> pieces = StrUtils.splitSmart(path, '/');
+    String blobName = null;
+    if(pieces.size()>=3) blobName = pieces.get(2);
+
+    if("POST".equals(httpMethod)) {
+      if (blobName == null || blobName.isEmpty()) {
+        rsp.add("error","Name not found");
+        return;
+      }
+      if(req.getContentStreams() == null )  {
+        rsp.add("error","No stream");
+        return;
+      }
+
+
+      for (ContentStream stream : req.getContentStreams()) {
+        ByteBuffer payload = SimplePostTool.inputStreamToByteArray(stream.getStream(), maxSize);
+        MessageDigest m = MessageDigest.getInstance("MD5");
+        m.update(payload.array(),payload.position(),payload.limit());
+        String md5 = new BigInteger(1,m.digest()).toString(16);
+
+        TopDocs duplicate = req.getSearcher().search(new TermQuery(new Term("id", md5)),
1);
+        if(duplicate.totalHits >0){
+          rsp.add("error", "duplicate entry");
+          SolrParams solrParams = new MapSolrParams((Map) ZkNodeProps.makeMap(
+              "q", "id:" + md5,
+              "fl", "id,size,version,timestamp,blobName"));
+          try(LocalSolrQueryRequest r = new LocalSolrQueryRequest(req.getCore(), solrParams))
{
+            req.getCore().getRequestHandler(null).handleRequest(r, rsp);
+          }
+          return;
+        }
+
+        TopFieldDocs docs = req.getSearcher().search(new TermQuery(new Term("blobName", blobName)),
+            null, 1, new Sort(new SortField("version", SortField.Type.LONG, true)));
+
+        long version = 0;
+        if(docs.totalHits >0){
+          StoredDocument doc = req.getSearcher().doc(docs.scoreDocs[0].doc);
+          Number n = doc.getField("version").numericValue();
+          version = n.longValue();
+        }
+
+        indexMap(req, makeMap(
+            "id", md5,
+            "blobName", blobName,
+            "version", ++version,
+            "timestamp", new Date(),
+            "size", payload.limit(),
+            "blob", payload));
+        break;
+      }
+
+    } else {
+      int version =-1;
+      if(pieces.size()>3){
+        try {
+          version = Integer.parseInt(pieces.get(3));
+        } catch (NumberFormatException e) {
+          rsp.add("error", "Invalid version" + pieces.get(3));
+          return;
+        }
+
+      }
+      if(ReplicationHandler.FILE_STREAM.equals(req.getParams().get(CommonParams.WT))){
+        if(blobName == null ){
+          throw new SolrException(SolrException.ErrorCode.NOT_FOUND, "Please send the request
in the format /blob/<blobName>/<version>");
+        } else {
+          String q = "blobName:{0}";
+          if(version!=-1) q+= " AND version:{1}";
+          QParser qparser =  QParser.getParser(MessageFormat.format(q,blobName,version) ,
"lucene", req);
+          final TopDocs docs = req.getSearcher().search(qparser.parse(), 1, new Sort( new
SortField("version", SortField.Type.LONG, true)));
+          if(docs.totalHits>0){
+            rsp.add(ReplicationHandler.FILE_STREAM, new SolrCore.RawWriter(){
+
+              @Override
+              public void write(OutputStream os) throws IOException {
+                StoredDocument doc = req.getSearcher().doc(docs.scoreDocs[0].doc);
+                StorableField sf = doc.getField("blob");
+                FieldType fieldType = req.getSchema().getField("blob").getType();
+                ByteBuffer buf = (ByteBuffer) fieldType.toObject(sf);
+                if(buf == null){
+                  //should never happen unless a user wrote this document directly
+                  throw new SolrException(SolrException.ErrorCode.NOT_FOUND, "Invalid document
. No field called blob");
+                } else {
+                  os.write(buf.array(),0,buf.limit());
+                }
+              }
+            });
+
+          } else {
+            throw new SolrException(SolrException.ErrorCode.NOT_FOUND,
+                MessageFormat.format("Invalid combination of blobName {0} and version {1}",
blobName,version));
+          }
+
+        }
+      } else {
+        String q = "*:*";
+        if (blobName != null) q = "blobName" + ":" + blobName;
+        if (version > -1) q += " AND version:" + version;
+
+        SolrParams args = new MapSolrParams((Map) ZkNodeProps.makeMap(
+            "q", q,
+            "fl", "id,size,version,timestamp,blobName",
+            "sort", "version desc"));
+        try (LocalSolrQueryRequest r  = new LocalSolrQueryRequest(req.getCore(), args)){
+          req.getCore().getRequestHandler(null).handleRequest(r, rsp);
+        }
+      }
+    }
+  }
+
+  public static void indexMap(SolrQueryRequest req, Map<String, Object> doc) throws
IOException {
+    SolrInputDocument solrDoc = new SolrInputDocument();
+    for (Map.Entry<String, Object> e : doc.entrySet()) solrDoc.addField(e.getKey(),e.getValue());
+    UpdateRequestProcessorChain processorChain = req.getCore().getUpdateProcessingChain(req.getParams().get(UpdateParams.UPDATE_CHAIN));
+    UpdateRequestProcessor processor = processorChain.createProcessor(req,null);
+    AddUpdateCommand cmd = new AddUpdateCommand(req);
+    cmd.commitWithin =1;
+    cmd.solrDoc = solrDoc;
+    processor.processAdd(cmd);
+
+  }
+
+  @Override
+  public SolrRequestHandler getSubHandler(String subPath) {
+    if(StrUtils.splitSmart(subPath,'/').size()>4)  return null;
+    return this;
+  }
+
+
+//////////////////////// SolrInfoMBeans methods //////////////////////
+
+  @Override
+  public String getDescription() {
+    return "Load Jars into a system index";
+  }
+
+  public static final String SCHEMA = "<?xml version='1.0' ?>\n" +
+      "<schema name='_system collection or core' version='1.1'>\n" +
+      "  <fieldtype name='string'  class='solr.StrField' sortMissingLast='true' omitNorms='true'/>\n"
+
+      "  <fieldType name='long' class='solr.TrieLongField' precisionStep='0' positionIncrementGap='0'/>\n"
+
+      "  <fieldType name='bytes' class='solr.BinaryField'/>\n" +
+      "  <fieldType name='date' class='solr.TrieDateField'/>\n" +
+      "  <field name='id'   type='string'   indexed='true'  stored='true'  multiValued='false'
required='true'/>\n" +
+      "  <field name='blob'      type='bytes'   indexed='false' stored='true'  multiValued='false'
/>\n" +
+      "  <field name='size'      type='long'   indexed='false' stored='true'  multiValued='false'
/>\n" +
+      "  <field name='version'   type='long'     indexed='true'  stored='true'  multiValued='false'
/>\n" +
+      "  <field name='timestamp'   type='date'   indexed='true'  stored='true'  multiValued='false'
/>\n" +
+      "  <field name='blobName'      type='string'   indexed='true'  stored='true'  multiValued='false'
/>\n" +
+      "  <field name='_version_' type='long'     indexed='true'  stored='true'/>\n"
+
+      "  <uniqueKey>id</uniqueKey>\n" +
+      "</schema>" ;
+
+  public static final String CONF = "<?xml version='1.0' ?>\n" +
+      "<config>\n" +
+      "<luceneMatchVersion>LATEST</luceneMatchVersion>\n" +
+      "<directoryFactory name='DirectoryFactory' class='${solr.directoryFactory:solr.StandardDirectoryFactory}'/>\n"
+
+      "<updateHandler class='solr.DirectUpdateHandler2'>\n" +
+      "  <updateLog>\n" +
+      "    <str name='dir'>${solr.ulog.dir:}</str>\n" +
+      "  </updateLog>\n" +
+      "</updateHandler>\n" +
+      "<requestHandler name='standard' class='solr.StandardRequestHandler' default='true'
/>\n" +
+      "<requestHandler name='/analysis/field' startup='lazy' class='solr.FieldAnalysisRequestHandler'
/>\n" +
+      "<requestHandler name='/blob' class='solr.BlobHandler'>\n" +
+      "  <lst name='invariants'>\n"+
+           "<str name='maxSize'>${blob.max.size.mb:5}</str>\n"+
+         "</lst>\n"+
+      "</requestHandler>\n" +
+      "</config>" ;
+
+  @Override
+  public void init(PluginInfo info) {
+    super.init(info.initArgs);
+    if(info.initArgs != null ){
+      NamedList invariants = (NamedList) info.initArgs.get(PluginInfo.INVARIANTS);
+      if(invariants != null){
+        Object o = invariants.get("maxSize");
+        if(o != null){
+          maxSize = Long.parseLong(String.valueOf(o));
+          maxSize = maxSize*1024*1024;
+        }
+      }
+
+    }
+  }
+}

Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java?rev=1644095&r1=1644094&r2=1644095&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java Tue
Dec  9 15:58:32 2014
@@ -860,7 +860,6 @@ public class ReplicationHandler extends
   @SuppressWarnings("unchecked")
   public void inform(SolrCore core) {
     this.core = core;
-    registerFileStreamResponseWriter();
     registerCloseHook();
     Object nbtk = initArgs.get(NUMBER_BACKUPS_TO_KEEP_INIT_PARAM);
     if(nbtk!=null) {
@@ -1010,34 +1009,6 @@ public class ReplicationHandler extends
   }
 
   /**
-   * A ResponseWriter is registered automatically for wt=filestream This response writer
is used to transfer index files
-   * in a block-by-block manner within the same HTTP response.
-   */
-  private void registerFileStreamResponseWriter() {
-    core.registerResponseWriter(FILE_STREAM, new BinaryQueryResponseWriter() {
-      @Override
-      public void write(OutputStream out, SolrQueryRequest request, SolrQueryResponse resp)
throws IOException {
-        DirectoryFileStream stream = (DirectoryFileStream) resp.getValues().get(FILE_STREAM);
-        stream.write(out);
-      }
-
-      @Override
-      public void write(Writer writer, SolrQueryRequest request, SolrQueryResponse response)
{
-        throw new RuntimeException("This is a binary writer , Cannot write to a characterstream");
-      }
-
-      @Override
-      public String getContentType(SolrQueryRequest request, SolrQueryResponse response)
{
-        return BinaryResponseParser.BINARY_CONTENT_TYPE;
-      }
-
-      @Override
-      public void init(NamedList args) { /*no op*/ }
-    });
-
-  }
-
-  /**
    * Register a listener for postcommit/optimize
    *
    * @param snapshoot do a snapshoot
@@ -1099,7 +1070,7 @@ public class ReplicationHandler extends
   /**This class is used to read and send files in the lucene index
    *
    */
-  private class DirectoryFileStream {
+  private class DirectoryFileStream implements SolrCore.RawWriter {
     protected SolrParams params;
 
     protected FastOutputStream fos;

Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java?rev=1644095&r1=1644094&r2=1644095&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java Tue
Dec  9 15:58:32 2014
@@ -21,6 +21,7 @@ import org.apache.solr.common.SolrExcept
 import org.apache.solr.common.params.SolrParams;
 import org.apache.solr.common.util.NamedList;
 import org.apache.solr.common.util.SimpleOrderedMap;
+import org.apache.solr.core.RequestHandlers;
 import org.apache.solr.core.SolrCore;
 import org.apache.solr.core.SolrInfoMBean;
 import org.apache.solr.request.SolrQueryRequest;
@@ -218,6 +219,9 @@ public abstract class RequestHandlerBase
           String firstPart = handlerName.substring(0, idx);
           handler = reqHandlers.get(firstPart);
           if (handler == null) continue;
+          if(handler instanceof RequestHandlers.LazyRequestHandlerWrapper) {
+            handler = ((RequestHandlers.LazyRequestHandlerWrapper)handler).getWrappedHandler();
+          }
           if (handler instanceof NestedRequestHandler) {
             return ((NestedRequestHandler) handler).getSubHandler(handlerName.substring(idx));
           }

Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java?rev=1644095&r1=1644094&r2=1644095&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
(original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
Tue Dec  9 15:58:32 2014
@@ -64,6 +64,7 @@ import static org.apache.solr.common.par
 import static org.apache.solr.common.params.CollectionParams.CollectionAction.SPLITSHARD;
 
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Iterator;
@@ -91,6 +92,7 @@ import org.apache.solr.common.cloud.DocC
 import org.apache.solr.common.cloud.ImplicitDocRouter;
 import org.apache.solr.common.cloud.Replica;
 import org.apache.solr.common.cloud.Slice;
+import org.apache.solr.common.cloud.SolrZkClient;
 import org.apache.solr.common.cloud.ZkCoreNodeProps;
 import org.apache.solr.common.cloud.ZkNodeProps;
 import org.apache.solr.common.cloud.ZkStateReader;
@@ -102,9 +104,11 @@ import org.apache.solr.common.params.Sol
 import org.apache.solr.common.util.NamedList;
 import org.apache.solr.common.util.SimpleOrderedMap;
 import org.apache.solr.core.CoreContainer;
+import org.apache.solr.handler.BlobHandler;
 import org.apache.solr.handler.RequestHandlerBase;
 import org.apache.solr.request.SolrQueryRequest;
 import org.apache.solr.response.SolrQueryResponse;
+import org.apache.zookeeper.CreateMode;
 import org.apache.zookeeper.KeeperException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -720,7 +724,7 @@ public class CollectionsHandler extends
       throw new SolrException(ErrorCode.BAD_REQUEST,
           "Collection name is required to create a new collection");
     }
-    
+
     Map<String,Object> props = ZkNodeProps.makeMap(
         Overseer.QUEUE_OPERATION,
         CREATE.toLower(),
@@ -738,12 +742,38 @@ public class CollectionsHandler extends
          AUTO_ADD_REPLICAS,
         "router.");
 
+    if(SYSTEM_COLL.equals(name)){
+      //We must always create asystem collection with only a single shard
+      props.put(NUM_SLICES,1);
+      props.remove(SHARDS_PROP);
+      createSysConfigSet();
+
+    }
     copyPropertiesIfNotNull(req.getParams(), props);
 
     ZkNodeProps m = new ZkNodeProps(props);
     handleResponse(CREATE.toLower(), m, rsp);
   }
 
+  private void createSysConfigSet() throws KeeperException, InterruptedException {
+    SolrZkClient zk = coreContainer.getZkController().getZkStateReader().getZkClient();
+    createNodeIfNotExists(zk,ZkStateReader.CONFIGS_ZKNODE+"/"+SYSTEM_COLL, null);
+    createNodeIfNotExists(zk,ZkStateReader.CONFIGS_ZKNODE+"/"+SYSTEM_COLL+"/schema.xml",
BlobHandler.SCHEMA.replaceAll("'","\"").getBytes(StandardCharsets.UTF_8));
+    createNodeIfNotExists(zk, ZkStateReader.CONFIGS_ZKNODE + "/" + SYSTEM_COLL + "/solrconfig.xml",
BlobHandler.CONF.replaceAll("'", "\"").getBytes(StandardCharsets.UTF_8));
+  }
+
+  public static void createNodeIfNotExists(SolrZkClient zk, String path, byte[] data) throws
KeeperException, InterruptedException {
+    if(!zk.exists(path, true)){
+      //create the config znode
+      try {
+        zk.create(path,data, CreateMode.PERSISTENT,true);
+      } catch (KeeperException.NodeExistsException e) {
+        //no problem . race condition. carry on the good work
+      }
+    }
+  }
+
+
   private void handleRemoveReplica(SolrQueryRequest req, SolrQueryResponse rsp) throws KeeperException,
InterruptedException {
     log.info("Remove replica: " + req.getParamString());
     req.getParams().required().check(COLLECTION_PROP, SHARD_ID_PROP, "replica");
@@ -928,5 +958,6 @@ public class CollectionsHandler extends
   public String getDescription() {
     return "Manage SolrCloud Collections";
   }
+  public static final String SYSTEM_COLL =".system";
 
 }

Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/request/SolrQueryRequestBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/request/SolrQueryRequestBase.java?rev=1644095&r1=1644094&r2=1644095&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/request/SolrQueryRequestBase.java
(original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/request/SolrQueryRequestBase.java
Tue Dec  9 15:58:32 2014
@@ -24,6 +24,7 @@ import org.apache.solr.common.params.Sol
 import org.apache.solr.common.util.ContentStream;
 import org.apache.solr.core.SolrCore;
 
+import java.io.Closeable;
 import java.util.Map;
 import java.util.HashMap;
 
@@ -40,7 +41,7 @@ import java.util.HashMap;
  *
  *
  */
-public abstract class SolrQueryRequestBase implements SolrQueryRequest {
+public abstract class SolrQueryRequestBase implements SolrQueryRequest, Closeable {
   protected final SolrCore core;
   protected final SolrParams origParams;
   protected volatile IndexSchema schema;

Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java?rev=1644095&r1=1644094&r2=1644095&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java Tue
Dec  9 15:58:32 2014
@@ -681,10 +681,7 @@ public class SolrRequestParsers
         if (ServletFileUpload.isMultipartContent(req)) {
           return multipart.parseParamsAndFillStreams(req, streams);
         }
-        if (req.getContentType() != null) {
-          return raw.parseParamsAndFillStreams(req, streams);
-        }
-        throw new SolrException(ErrorCode.UNSUPPORTED_MEDIA_TYPE, "Must specify a Content-Type
header with POST requests");
+        return raw.parseParamsAndFillStreams(req, streams);
       }
       throw new SolrException(ErrorCode.BAD_REQUEST, "Unsupported method: " + method + "
for request " + req);
     }

Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/util/SimplePostTool.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/util/SimplePostTool.java?rev=1644095&r1=1644094&r2=1644095&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/util/SimplePostTool.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/util/SimplePostTool.java Tue Dec 
9 15:58:32 2014
@@ -32,6 +32,8 @@ import java.net.MalformedURLException;
 import java.net.ProtocolException;
 import java.net.URL;
 import java.net.URLEncoder;
+import java.nio.BufferOverflowException;
+import java.nio.ByteBuffer;
 import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
 import java.text.SimpleDateFormat;
@@ -60,6 +62,7 @@ import javax.xml.xpath.XPathExpression;
 import javax.xml.xpath.XPathExpressionException;
 import javax.xml.xpath.XPathFactory;
 
+import org.apache.zookeeper.server.ByteBufferInputStream;
 import org.w3c.dom.Document;
 import org.w3c.dom.Node;
 import org.w3c.dom.NodeList;
@@ -581,14 +584,14 @@ public class SimplePostTool {
           URL postUrl = new URL(appendParam(solrUrl.toString(), 
               "literal.id="+URLEncoder.encode(u.toString(),"UTF-8") +
               "&literal.url="+URLEncoder.encode(u.toString(),"UTF-8")));
-          boolean success = postData(new ByteArrayInputStream(result.content), null, out,
result.contentType, postUrl);
+          boolean success = postData(new ByteArrayInputStream(result.content.array(), result.content.arrayOffset(),result.content.limit()
), null, out, result.contentType, postUrl);
           if (success) {
             info("POSTed web resource "+u+" (depth: "+level+")");
             Thread.sleep(delay * 1000);
             numPages++;
             // Pull links from HTML pages only
             if(recursive > level && result.contentType.equals("text/html")) {
-              Set<URL> children = pageFetcher.getLinksFromWebPage(u, new ByteArrayInputStream(result.content),
result.contentType, postUrl);
+              Set<URL> children = pageFetcher.getLinksFromWebPage(u, new ByteBufferInputStream(result.content),
result.contentType, postUrl);
               subStack.addAll(children);
             }
           } else {
@@ -609,23 +612,35 @@ public class SimplePostTool {
     }
     return numPages;    
   }
+  public static class BAOS extends ByteArrayOutputStream {
+    public ByteBuffer getByteBuffer() {
+      return ByteBuffer.wrap(super.buf,0,super.count);
+    }
+  }
+  public static ByteBuffer inputStreamToByteArray(InputStream is) throws IOException {
+    return inputStreamToByteArray(is,Integer.MAX_VALUE);
+
+  }
 
   /**
    * Reads an input stream into a byte array
+   *
    * @param is the input stream
    * @return the byte array
    * @throws IOException If there is a low-level I/O error.
    */
-  protected byte[] inputStreamToByteArray(InputStream is) throws IOException {
-    ByteArrayOutputStream bos = new ByteArrayOutputStream();
+  public static ByteBuffer inputStreamToByteArray(InputStream is, long maxSize) throws IOException
{
+    BAOS bos =  new BAOS();
+    long sz = 0;
     int next = is.read();
     while (next > -1) {
-        bos.write(next);
-        next = is.read();
+      if(++sz > maxSize) throw new BufferOverflowException();
+      bos.write(next);
+      next = is.read();
     }
     bos.flush();
     is.close();
-    return bos.toByteArray();
+    return bos.getByteBuffer();
   }
 
   /**
@@ -1198,6 +1213,6 @@ public class SimplePostTool {
     int httpStatus = 200;
     String contentType = "text/html";
     URL redirectUrl = null;
-    byte[] content;
+    ByteBuffer content;
   }
 }

Added: lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/TestBlobHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/TestBlobHandler.java?rev=1644095&view=auto
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/TestBlobHandler.java (added)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/TestBlobHandler.java Tue Dec
 9 15:58:32 2014
@@ -0,0 +1,178 @@
+package org.apache.solr.handler;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.net.URLConnection;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.io.Closeables;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.entity.ByteArrayEntity;
+import org.apache.http.util.EntityUtils;
+import org.apache.solr.client.solrj.SolrServer;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.HttpSolrServer;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.response.CollectionAdminResponse;
+import org.apache.solr.cloud.AbstractFullDistribZkTestBase;
+import org.apache.solr.common.cloud.DocCollection;
+import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.core.ConfigOverlay;
+import org.apache.solr.update.DirectUpdateHandler2;
+import org.apache.solr.util.SimplePostTool;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.solr.core.ConfigOverlay.getObjectByPath;
+
+public class TestBlobHandler extends AbstractFullDistribZkTestBase {
+  static final Logger log =  LoggerFactory.getLogger(TestSolrConfigHandlerConcurrent.class);
+
+  private void doBlobHandlerTest() throws Exception {
+    SolrServer server = createNewSolrServer("", getBaseUrl((HttpSolrServer) clients.get(0)));
+
+    CollectionAdminResponse response1;
+    CollectionAdminRequest.Create createCollectionRequest = new CollectionAdminRequest.Create();
+    createCollectionRequest.setCollectionName(".system");
+    createCollectionRequest.setNumShards(1);
+    createCollectionRequest.setReplicationFactor(2);
+    response1 = createCollectionRequest.process(server);
+    assertEquals(0, response1.getStatus());
+    assertTrue(response1.isSuccess());
+    DocCollection sysColl = cloudClient.getZkStateReader().getClusterState().getCollection(".system");
+    Replica replica = sysColl.getActiveSlicesMap().values().iterator().next().getLeader();
+
+    String baseUrl = replica.getStr(ZkStateReader.BASE_URL_PROP);
+    String url = baseUrl + "/.system/config/requestHandler";
+    Map map = TestSolrConfigHandlerConcurrent.getAsMap(url, cloudClient);
+    assertNotNull(map);
+    assertEquals("solr.BlobHandler", getObjectByPath(map, true, Arrays.asList(
+        "solrConfig",
+        "requestHandler",
+        "/blob",
+        "class")));
+
+    byte[] bytarr  = new byte[1024];
+    for (int i = 0; i < bytarr.length; i++) bytarr[i]= (byte) (i % 127);
+    byte[] bytarr2  = new byte[2048];
+    for (int i = 0; i < bytarr2.length; i++) bytarr2[i]= (byte) (i % 127);
+    postAndCheck(baseUrl, bytarr, 1);
+    postAndCheck(baseUrl, bytarr2, 2);
+
+    url = baseUrl + "/.system/blob/test/1";
+    map = TestSolrConfigHandlerConcurrent.getAsMap(url,cloudClient);
+    List l = (List) ConfigOverlay.getObjectByPath(map, false, Arrays.asList("response", "docs"));
+    assertNotNull(l);
+    map = (Map) l.get(0);
+    assertEquals(""+bytarr.length,String.valueOf(map.get("size")));
+
+    compareInputAndOutput(baseUrl+"/.system/blob/test?wt=filestream", bytarr2);
+    compareInputAndOutput(baseUrl+"/.system/blob/test/1?wt=filestream", bytarr);
+
+  }
+
+  @Override
+  public void tearDown() throws Exception {
+    super.tearDown();
+    System.clearProperty("numShards");
+    System.clearProperty("zkHost");
+
+    // insurance
+    DirectUpdateHandler2.commitOnClose = true;
+  }
+
+  private void postAndCheck(String baseUrl, byte[] bytes, int count) throws Exception {
+    postData(baseUrl, bytes);
+    String url;
+    Map map;
+    List l;
+    long startTime = System.nanoTime();
+    long maxTimeoutSeconds = 10;
+    while ( true) {
+      url = baseUrl + "/.system/blob/test";
+      map = TestSolrConfigHandlerConcurrent.getAsMap(url, cloudClient);
+      String numFound = String.valueOf(ConfigOverlay.getObjectByPath(map, false, Arrays.asList("response",
"numFound")));
+      if(!(""+count).equals(numFound)) {
+        if (TimeUnit.SECONDS.convert(System.nanoTime() - startTime, TimeUnit.NANOSECONDS)
< maxTimeoutSeconds) {
+          Thread.sleep(100);
+          continue;
+        }
+      }
+      l = (List) ConfigOverlay.getObjectByPath(map, false, Arrays.asList("response", "docs"));
+      assertNotNull(l);
+      map = (Map) l.get(0);
+      assertEquals("" + bytes.length, String.valueOf(map.get("size")));
+      break;
+    }
+  }
+
+  private void compareInputAndOutput(String url, byte[] bytarr) throws IOException {
+
+    HttpClient httpClient = cloudClient.getLbServer().getHttpClient();
+
+    HttpGet httpGet = new HttpGet(url);
+    HttpResponse entity = httpClient.execute(httpGet);
+    ByteBuffer b = SimplePostTool.inputStreamToByteArray(entity.getEntity().getContent());
+    try {
+      assertEquals(b.limit(), bytarr.length);
+      for (int i = 0; i < bytarr.length; i++) {
+        assertEquals(b.get(i), bytarr[i]);
+      }
+    } finally {
+      httpGet.releaseConnection();
+    }
+
+  }
+
+  private String postData(String baseUrl, byte[] bytarr) throws IOException {
+    HttpPost httpPost = null;
+    HttpEntity entity;
+    String response;
+    try {
+      httpPost = new HttpPost(baseUrl+"/.system/blob/test");
+      httpPost.setHeader("Content-Type","application/octet-stream");
+      httpPost.setEntity(new ByteArrayEntity(bytarr));
+      entity = cloudClient.getLbServer().getHttpClient().execute(httpPost).getEntity();
+      return EntityUtils.toString(entity, StandardCharsets.UTF_8);
+    } finally {
+      httpPost.releaseConnection();
+    }
+  }
+
+  @Override
+  public void doTest() throws Exception {
+    doBlobHandlerTest();
+
+  }
+}

Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/TestSolrConfigHandlerConcurrent.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/TestSolrConfigHandlerConcurrent.java?rev=1644095&r1=1644094&r2=1644095&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/TestSolrConfigHandlerConcurrent.java
(original)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/TestSolrConfigHandlerConcurrent.java
Tue Dec  9 15:58:32 2014
@@ -201,6 +201,7 @@ public class TestSolrConfigHandlerConcur
       return (Map) ObjectBuilder.getVal(new JSONParser(new StringReader(response)));
     } finally {
       EntityUtils.consumeQuietly(entity);
+      get.releaseConnection();
     }
   }
 }

Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/servlet/SolrRequestParserTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/servlet/SolrRequestParserTest.java?rev=1644095&r1=1644094&r2=1644095&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/servlet/SolrRequestParserTest.java
(original)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/servlet/SolrRequestParserTest.java
Tue Dec  9 15:58:32 2014
@@ -17,6 +17,7 @@
 
 package org.apache.solr.servlet;
 
+import static org.easymock.EasyMock.anyObject;
 import static org.easymock.EasyMock.createMock;
 import static org.easymock.EasyMock.expect;
 import static org.easymock.EasyMock.replay;
@@ -386,15 +387,14 @@ public class SolrRequestParserTest exten
     expect(request.getMethod()).andReturn("POST").anyTimes();
     expect(request.getContentType()).andReturn(null).anyTimes();
     expect(request.getQueryString()).andReturn(null).anyTimes();
+    expect(request.getHeader(anyObject())).andReturn(null).anyTimes();
     replay(request);
 
     SolrRequestParsers parsers = new SolrRequestParsers(h.getCore().getSolrConfig());
     try {
       parsers.parse(h.getCore(), "/select", request);
-      fail("should throw SolrException");
     } catch (SolrException e) {
-      assertTrue(e.getMessage().startsWith("Must specify a Content-Type header with POST
requests"));
-      assertEquals(415, e.code());
+      fail("should not throw SolrException");
     }
   }
 }

Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/util/SimplePostToolTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/util/SimplePostToolTest.java?rev=1644095&r1=1644094&r2=1644095&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/util/SimplePostToolTest.java (original)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/util/SimplePostToolTest.java Tue Dec
 9 15:58:32 2014
@@ -23,6 +23,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.net.MalformedURLException;
 import java.net.URL;
+import java.nio.ByteBuffer;
 import java.nio.charset.StandardCharsets;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -225,7 +226,7 @@ public class SimplePostToolTest extends
       }
       res.httpStatus = 200;
       res.contentType = "text/html";
-      res.content = htmlMap.get(u.toString()).getBytes(StandardCharsets.UTF_8);
+      res.content = ByteBuffer.wrap( htmlMap.get(u.toString()).getBytes(StandardCharsets.UTF_8));
       return res;
     }
     



Mime
View raw message