lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sh...@apache.org
Subject svn commit: r1458924 [11/13] - in /lucene/dev/branches/lucene4258: ./ dev-tools/ dev-tools/eclipse/dot.settings/ dev-tools/maven/ dev-tools/maven/solr/solrj/src/java/ dev-tools/scripts/ lucene/ lucene/analysis/ lucene/analysis/common/ lucene/analysis/c...
Date Wed, 20 Mar 2013 16:27:23 GMT
Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/CachingDirectoryFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/CachingDirectoryFactory.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/CachingDirectoryFactory.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/CachingDirectoryFactory.java Wed Mar 20 16:27:13 2013
@@ -20,10 +20,14 @@ package org.apache.solr.core;
 import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.HashMap;
+import java.util.HashSet;
+import java.util.IdentityHashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Set;
 
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext.Context;
@@ -42,13 +46,42 @@ import org.slf4j.LoggerFactory;
  * per path. Most DirectoryFactory implementations will want to extend this
  * class and simply implement {@link DirectoryFactory#create(String, DirContext)}.
  * 
+ * This is an expert class and these API's are subject to change.
+ * 
  */
 public abstract class CachingDirectoryFactory extends DirectoryFactory {
   protected class CacheValue {
-    public Directory directory;
+    final public String path;
+    final public Directory directory;
+    
+    // use the setter!
+    private boolean deleteOnClose = false;
+    
+    public CacheValue(String path, Directory directory) {
+      this.path = path;
+      this.directory = directory;
+      this.closeEntries.add(this);
+    }
     public int refCnt = 1;
-    public String path;
+    // if we are latestForPath, I'm currently using my path
+    // otherwise a new Directory instance is using my path
+    // and I must be manipulated by Directory
+    public boolean latestForPath = false;
+    // has close(Directory) been called on this?
+    public boolean closeDirectoryCalled = false;
     public boolean doneWithDir = false;
+    private boolean deleteAfterCoreClose = false;
+    public Set<CacheValue> removeEntries = new HashSet<CacheValue>();
+    public Set<CacheValue> closeEntries = new HashSet<CacheValue>();
+
+    public void setDeleteOnClose(boolean deleteOnClose, boolean deleteAfterCoreClose) {
+      if (deleteOnClose) {
+        removeEntries.add(this);
+      }
+      this.deleteOnClose = deleteOnClose;
+      this.deleteAfterCoreClose = deleteAfterCoreClose;
+    }
+    
     @Override
     public String toString() {
       return "CachedDir<<" + directory.toString() + ";refCount=" + refCnt + ";path=" + path + ";done=" + doneWithDir + ">>";
@@ -60,9 +93,11 @@ public abstract class CachingDirectoryFa
   
   protected Map<String,CacheValue> byPathCache = new HashMap<String,CacheValue>();
   
-  protected Map<Directory,CacheValue> byDirectoryCache = new HashMap<Directory,CacheValue>();
+  protected Map<Directory,CacheValue> byDirectoryCache = new IdentityHashMap<Directory,CacheValue>();
   
   protected Map<Directory,List<CloseListener>> closeListeners = new HashMap<Directory,List<CloseListener>>();
+  
+  protected Set<CacheValue> removeEntries = new HashSet<CacheValue>();
 
   private Double maxWriteMBPerSecFlush;
 
@@ -123,7 +158,9 @@ public abstract class CachingDirectoryFa
   public void close() throws IOException {
     synchronized (this) {
       this.closed = true;
-      for (CacheValue val : byDirectoryCache.values()) {
+      Collection<CacheValue> values = new ArrayList<CacheValue>();
+      values.addAll(byDirectoryCache.values());
+      for (CacheValue val : values) {
         try {
           // if there are still refs out, we have to wait for them
           int cnt = 0;
@@ -135,7 +172,15 @@ public abstract class CachingDirectoryFa
               break;
             }
           }
-          
+          assert val.refCnt == 0 : val.refCnt;
+        } catch (Throwable t) {
+          SolrException.log(log, "Error closing directory", t);
+        }
+      }
+      
+      values = byDirectoryCache.values();
+      for (CacheValue val : values) {
+        try {
           assert val.refCnt == 0 : val.refCnt;
           log.info("Closing directory when closing factory: " + val.path);
           closeDirectory(val);
@@ -143,8 +188,14 @@ public abstract class CachingDirectoryFa
           SolrException.log(log, "Error closing directory", t);
         }
       }
+      
       byDirectoryCache.clear();
       byPathCache.clear();
+      
+      for (CacheValue val : removeEntries) {
+        log.info("Removing directory: " + val.path);
+        removeDirectory(val);
+      }
     }
   }
   
@@ -166,7 +217,13 @@ public abstract class CachingDirectoryFa
         closeDirectory(cacheValue);
         
         byDirectoryCache.remove(directory);
-        byPathCache.remove(cacheValue.path);
+        
+        // if it's been closed, it's path is now
+        // owned by another Directory instance
+        if (!cacheValue.latestForPath) {
+          byPathCache.remove(cacheValue.path);
+          cacheValue.latestForPath = true;
+        }
       }
     }
   }
@@ -182,13 +239,56 @@ public abstract class CachingDirectoryFa
         }
       }
     }
-    try {
-      log.info("Closing directory: " + cacheValue.path);
-      cacheValue.directory.close();
-    } catch (Throwable t) {
-      SolrException.log(log, "Error closing directory", t);
+    
+    cacheValue.closeDirectoryCalled = true;
+    
+    if (cacheValue.deleteOnClose) {
+      
+      // see if we are a subpath
+      Collection<CacheValue> values = byPathCache.values();
+      
+      Collection<CacheValue> cacheValues = new ArrayList<CacheValue>();
+      cacheValues.addAll(values);
+      cacheValues.remove(cacheValue);
+      for (CacheValue otherCacheValue : cacheValues) {
+        // if we are a parent path and all our sub children are not already closed,
+        // get a sub path to close us later
+        if (otherCacheValue.path.startsWith(cacheValue.path) && !otherCacheValue.closeDirectoryCalled) {
+          // we let the sub dir remove and close us
+          if (!otherCacheValue.deleteAfterCoreClose && cacheValue.deleteAfterCoreClose) {
+            otherCacheValue.deleteAfterCoreClose = true;
+          }
+          otherCacheValue.removeEntries.addAll(cacheValue.removeEntries);
+          otherCacheValue.closeEntries.addAll(cacheValue.closeEntries);
+          cacheValue.closeEntries.clear();
+          break;
+        }
+      }
+    }
+    
+    for (CacheValue val : cacheValue.removeEntries) {
+      if (!val.deleteAfterCoreClose) {
+        try {
+          log.info("Removing directory: " + val.path);
+          removeDirectory(val);
+        } catch (Throwable t) {
+          SolrException.log(log, "Error removing directory", t);
+        }
+      } else {
+        removeEntries.add(val);
+      }
     }
     
+    for (CacheValue val : cacheValue.closeEntries) {
+      try {
+        log.info("Closing directory: " + val.path);
+        val.directory.close();
+      } catch (Throwable t) {
+        SolrException.log(log, "Error closing directory", t);
+      }
+      
+    }
+
     if (listeners != null) {
       for (CloseListener listener : listeners) {
         try {
@@ -199,12 +299,12 @@ public abstract class CachingDirectoryFa
       }
     }
   }
-  
+
   @Override
   protected abstract Directory create(String path, DirContext dirContext) throws IOException;
   
   @Override
-  public boolean exists(String path) {
+  public boolean exists(String path) throws IOException {
     // back compat behavior
     File dirFile = new File(path);
     return dirFile.canRead() && dirFile.list().length > 0;
@@ -231,7 +331,7 @@ public abstract class CachingDirectoryFa
   @Override
   public final Directory get(String path,  DirContext dirContext, String rawLockType, boolean forceNew)
       throws IOException {
-    String fullPath = new File(path).getAbsolutePath();
+    String fullPath = normalize(path);
     synchronized (this) {
       if (closed) {
         throw new RuntimeException("Already closed");
@@ -247,18 +347,14 @@ public abstract class CachingDirectoryFa
           // we make a quick close attempt,
           // otherwise this should be closed
           // when whatever is using it, releases it
-          
           if (cacheValue.refCnt == 0) {
-            try {
-              // the following will decref, so
-              // first incref
-              cacheValue.refCnt++;
-              close(cacheValue.directory);
-            } catch (IOException e) {
-              SolrException.log(log, "Error closing directory", e);
-            }
+            closeDirectory(cacheValue);
           }
           
+          // close the entry, it will be owned by the new dir
+          // we count on it being released by directory
+          cacheValue.latestForPath = true;
+          
         }
       }
       
@@ -267,11 +363,9 @@ public abstract class CachingDirectoryFa
         
         directory = rateLimit(directory);
         
-        CacheValue newCacheValue = new CacheValue();
-        newCacheValue.directory = directory;
-        newCacheValue.path = fullPath;
+        CacheValue newCacheValue = new CacheValue(fullPath, directory);
         
-        injectLockFactory(directory, path, rawLockType);
+        injectLockFactory(directory, fullPath, rawLockType);
         
         byDirectoryCache.put(directory, newCacheValue);
         byPathCache.put(fullPath, newCacheValue);
@@ -345,6 +439,38 @@ public abstract class CachingDirectoryFa
     close(directory);
   }
   
+  @Override
+  public void remove(String path) throws IOException {
+    remove(path, false);
+  }
+  
+  @Override
+  public void remove(Directory dir) throws IOException {
+    remove(dir, false);
+  }
+  
+  @Override
+  public void remove(String path, boolean deleteAfterCoreClose) throws IOException {
+    synchronized (this) {
+      CacheValue val = byPathCache.get(normalize(path));
+      if (val == null) {
+        throw new IllegalArgumentException("Unknown directory " + path);
+      }
+      val.setDeleteOnClose(true, deleteAfterCoreClose);
+    }
+  }
+  
+  @Override
+  public void remove(Directory dir, boolean deleteAfterCoreClose) throws IOException {
+    synchronized (this) {
+      CacheValue val = byDirectoryCache.get(dir);
+      if (val == null) {
+        throw new IllegalArgumentException("Unknown directory " + dir);
+      }
+      val.setDeleteOnClose(true, deleteAfterCoreClose);
+    }
+  }
+  
   private static Directory injectLockFactory(Directory dir, String lockPath,
       String rawLockType) throws IOException {
     if (null == rawLockType) {
@@ -372,4 +498,21 @@ public abstract class CachingDirectoryFa
     }
     return dir;
   }
+  
+  protected void removeDirectory(CacheValue cacheValue) throws IOException {
+    empty(cacheValue.directory);
+  }
+  
+  @Override
+  public String normalize(String path) throws IOException {
+    path = stripTrailingSlash(path);
+    return path;
+  }
+  
+  private String stripTrailingSlash(String path) {
+    if (path.endsWith("/")) {
+      path = path.substring(0, path.length() - 1);
+    }
+    return path;
+  }
 }

Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/ConfigSolrXmlBackCompat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/ConfigSolrXmlBackCompat.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/ConfigSolrXmlBackCompat.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/ConfigSolrXmlBackCompat.java Wed Mar 20 16:27:13 2013
@@ -25,6 +25,7 @@ import org.apache.solr.schema.IndexSchem
 import org.apache.solr.util.DOMUtil;
 import org.apache.solr.util.PropertiesUtil;
 import org.apache.solr.util.SystemIdResolver;
+import org.apache.solr.util.plugin.PluginInfoInitialized;
 import org.apache.zookeeper.KeeperException;
 import org.w3c.dom.Document;
 import org.w3c.dom.NamedNodeMap;
@@ -132,9 +133,16 @@ public class ConfigSolrXmlBackCompat ext
       m.put("class", HttpShardHandlerFactory.class.getName());
       info = new PluginInfo("shardHandlerFactory", m, null, Collections.<PluginInfo>emptyList());
     }
-    HttpShardHandlerFactory fac = new HttpShardHandlerFactory();
-    if (info != null) {
-      fac.init(info);
+
+    ShardHandlerFactory fac;
+    try {
+       fac = getResourceLoader().findClass(info.className, ShardHandlerFactory.class).newInstance();
+    } catch (Exception e) {
+      throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+                              "Error instantiating shardHandlerFactory class " + info.className);
+    }
+    if (fac instanceof PluginInfoInitialized) {
+      ((PluginInfoInitialized) fac).init(info);
     }
     return fac;
   }

Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/CoreContainer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/CoreContainer.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/CoreContainer.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/CoreContainer.java Wed Mar 20 16:27:13 2013
@@ -377,7 +377,7 @@ public class CoreContainer
         cfg = new ConfigSolrXmlBackCompat(loader, null, is, null, false);
         this.cfg = new ConfigSolrXmlBackCompat(loader, (ConfigSolrXmlBackCompat)cfg);
       } else {
-        cfg = new SolrProperties(this, is, fileName);
+        cfg = new SolrProperties(this, loader, is, fileName);
         this.cfg = new SolrProperties(this, loader, (SolrProperties)cfg);
       }
     } catch (Exception e) {
@@ -497,7 +497,6 @@ public class CoreContainer
         TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(),
         new DefaultSolrThreadFactory("coreLoadExecutor"));
     try {
-
       CompletionService<SolrCore> completionService = new ExecutorCompletionService<SolrCore>(
           coreLoadExecutor);
       Set<Future<SolrCore>> pending = new HashSet<Future<SolrCore>>();
@@ -588,7 +587,6 @@ public class CoreContainer
                 return c;
               }
             };
-
             pending.add(completionService.submit(task));
 
           } else {
@@ -607,7 +605,7 @@ public class CoreContainer
           Future<SolrCore> future = completionService.take();
           if (future == null) return;
           pending.remove(future);
-          
+
           try {
             SolrCore c = future.get();
             // track original names
@@ -680,7 +678,7 @@ public class CoreContainer
 
 
     try {
-      // First allow the closer thread to drain all the pending closes it can.
+      // First wake up the closer thread, it'll terminate almost immediately since it checks isShutDown.
       synchronized (coreMaps.getLocker()) {
         coreMaps.getLocker().notifyAll(); // wake up anyone waiting
       }
@@ -1032,53 +1030,55 @@ public class CoreContainer
    */
   public void reload(String name) {
     try {
+      name = checkDefault(name);
 
-      name= checkDefault(name);
       SolrCore core = coreMaps.getCore(name);
       if (core == null)
         throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "No such core: " + name );
 
-      CoreDescriptor cd = core.getCoreDescriptor();
-  
-      File instanceDir = new File(cd.getInstanceDir());
+      try {
+        coreMaps.waitAddPendingCoreOps(name);
+        CoreDescriptor cd = core.getCoreDescriptor();
 
-      log.info("Reloading SolrCore '{}' using instanceDir: {}", 
-               cd.getName(), instanceDir.getAbsolutePath());
-    
-      SolrResourceLoader solrLoader;
-      if(zkController == null) {
-        solrLoader = new SolrResourceLoader(instanceDir.getAbsolutePath(), libLoader, SolrProperties.getCoreProperties(instanceDir.getAbsolutePath(), cd));
-      } else {
-        try {
-          String collection = cd.getCloudDescriptor().getCollectionName();
-          zkController.createCollectionZkNode(cd.getCloudDescriptor());
+        File instanceDir = new File(cd.getInstanceDir());
+
+        log.info("Reloading SolrCore '{}' using instanceDir: {}",
+                 cd.getName(), instanceDir.getAbsolutePath());
+        SolrResourceLoader solrLoader;
+        if(zkController == null) {
+          solrLoader = new SolrResourceLoader(instanceDir.getAbsolutePath(), libLoader, SolrProperties.getCoreProperties(instanceDir.getAbsolutePath(), cd));
+        } else {
+          try {
+            String collection = cd.getCloudDescriptor().getCollectionName();
+            zkController.createCollectionZkNode(cd.getCloudDescriptor());
 
-          String zkConfigName = zkController.readConfigName(collection);
-          if (zkConfigName == null) {
-            log.error("Could not find config name for collection:" + collection);
+            String zkConfigName = zkController.readConfigName(collection);
+            if (zkConfigName == null) {
+              log.error("Could not find config name for collection:" + collection);
+              throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
+                                           "Could not find config name for collection:" + collection);
+            }
+            solrLoader = new ZkSolrResourceLoader(instanceDir.getAbsolutePath(), zkConfigName, libLoader,
+                SolrProperties.getCoreProperties(instanceDir.getAbsolutePath(), cd), zkController);
+          } catch (KeeperException e) {
+            log.error("", e);
+            throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
+                                         "", e);
+          } catch (InterruptedException e) {
+            // Restore the interrupted status
+            Thread.currentThread().interrupt();
+            log.error("", e);
             throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
-                                         "Could not find config name for collection:" + collection);
+                                         "", e);
           }
-          solrLoader = new ZkSolrResourceLoader(instanceDir.getAbsolutePath(), zkConfigName, libLoader,
-              SolrProperties.getCoreProperties(instanceDir.getAbsolutePath(), cd), zkController);
-        } catch (KeeperException e) {
-          log.error("", e);
-          throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
-                                       "", e);
-        } catch (InterruptedException e) {
-          // Restore the interrupted status
-          Thread.currentThread().interrupt();
-          log.error("", e);
-          throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
-                                       "", e);
         }
+        SolrCore newCore = core.reload(solrLoader, core);
+        // keep core to orig name link
+        coreMaps.removeCoreToOrigName(newCore, core);
+        registerCore(false, name, newCore, false);
+      } finally {
+        coreMaps.removeFromPendingOps(name);
       }
-      SolrCore newCore = core.reload(solrLoader, core);
-      // keep core to orig name link
-      coreMaps.removeCoreToOrigName(newCore, core);
-
-      registerCore(false, name, newCore, false);
-
       // :TODO: Java7...
       // http://docs.oracle.com/javase/7/docs/technotes/guides/language/catch-multiple.html
     } catch (Exception ex) {
@@ -1137,7 +1137,10 @@ public class CoreContainer
     // Do this in two phases since we don't want to lock access to the cores over a load.
     SolrCore core = coreMaps.getCoreFromAnyList(name);
 
-    if (core != null) return core;
+    if (core != null) {
+      core.open();
+      return core;
+    }
 
     // OK, it's not presently in any list, is it in the list of dynamic cores but not loaded yet? If so, load it.
     CoreDescriptor desc = coreMaps.getDynamicDescriptor(name);
@@ -1145,22 +1148,25 @@ public class CoreContainer
       return null;
     }
 
-    core = coreMaps.waitPendingCoreOps(name); // This will put an entry in pending core ops if the core isn't loaded
+    // This will put an entry in pending core ops if the core isn't loaded
+    core = coreMaps.waitAddPendingCoreOps(name);
 
     if (isShutDown) return null; // We're quitting, so stop. This needs to be after the wait above since we may come off
                                  // the wait as a consequence of shutting down.
-
-    if (core == null) {
-      try {
+    try {
+      if (core == null) {
         core = create(desc); // This should throw an error if it fails.
         core.open();
         registerCore(desc.isTransient(), name, core, false);
-      } catch (Exception ex) {
-        throw recordAndThrow(name, "Unable to create core: " + name, ex);
-      } finally {
-        coreMaps.releasePending(name);
+      } else {
+        core.open();
       }
+    } catch(Exception ex){
+      throw recordAndThrow(name, "Unable to create core: " + name, ex);
+    } finally {
+      coreMaps.removeFromPendingOps(name);
     }
+
     return core;
   }
 
@@ -1371,6 +1377,7 @@ public class CoreContainer
 // dynamicDescriptors
 //
 
+
 class CoreMaps {
 
   private static Object locker = new Object(); // for locking around manipulating any of the core maps.
@@ -1387,11 +1394,13 @@ class CoreMaps {
 
   private final CoreContainer container;
 
-  // It's a little clumsy to have two, but closing requires a SolrCore, whereas pending loads don't have a core.
-  private static final Set<String> pendingDynamicLoads = new TreeSet<String>();
-
-  // Holds cores from the time they're removed from the transient cache until after they're closed.
-  private static final List<SolrCore> pendingDynamicCloses = new ArrayList<SolrCore>();
+  // This map will hold objects that are being currently operated on. The core (value) may be null in the case of
+  // initial load. The rule is, never to any operation on a core that is currently being operated upon.
+  private static final Set<String> pendingCoreOps = new HashSet<String>();
+
+  // Due to the fact that closes happen potentially whenever anything is _added_ to the transient core list, we need
+  // to essentially queue them up to be handled via pendingCoreOps.
+  private static final List<SolrCore> pendingCloses = new ArrayList<SolrCore>();
 
   CoreMaps(CoreContainer container) {
     this.container = container;
@@ -1408,11 +1417,8 @@ class CoreMaps {
         protected boolean removeEldestEntry(Map.Entry<String, SolrCore> eldest) {
           if (size() > transientCacheSize) {
             synchronized (locker) {
-              SolrCore closeMe = eldest.getValue();
-              synchronized (locker) {
-                pendingDynamicCloses.add(closeMe);
-                locker.notifyAll(); // Wakes up closer thread too
-              }
+              pendingCloses.add(eldest.getValue()); // Essentially just queue this core up for closing.
+              locker.notifyAll(); // Wakes up closer thread too
             }
             return true;
           }
@@ -1433,11 +1439,11 @@ class CoreMaps {
   protected void clearMaps(ConfigSolr cfg) {
     List<String> coreNames;
     List<String> transientNames;
-    List<SolrCore> pendingClosers;
+    List<SolrCore> pendingToClose;
     synchronized (locker) {
       coreNames = new ArrayList(cores.keySet());
       transientNames = new ArrayList(transientCores.keySet());
-      pendingClosers = new ArrayList(pendingDynamicCloses);
+      pendingToClose = new ArrayList(pendingCloses);
     }
     for (String coreName : coreNames) {
       SolrCore core = cores.get(coreName);
@@ -1466,8 +1472,12 @@ class CoreMaps {
     transientCores.clear();
 
     // We might have some cores that we were _thinking_ about shutting down, so take care of those too.
-    for (SolrCore core : pendingClosers) {
-      core.close();
+    for (SolrCore core : pendingToClose) {
+      try {
+        core.close();
+      } catch (Throwable t) {
+        SolrException.log(CoreContainer.log, "Error shutting down core", t);
+      }
     }
 
   }
@@ -1610,10 +1620,9 @@ class CoreMaps {
   protected SolrCore getCoreFromAnyList(String name) {
     SolrCore core;
 
-    synchronized (locker) { // This one's OK, the core.open is just an increment
+    synchronized (locker) {
       core = cores.get(name);
       if (core != null) {
-        core.open();    // increment the ref count while still synchronized
         return core;
       }
 
@@ -1621,15 +1630,8 @@ class CoreMaps {
         return null; // Nobody even tried to define any transient cores, so we're done.
       }
       // Now look for already loaded transient cores.
-      core = transientCores.get(name);
-      if (core != null) {
-        core.open();  // Just increments ref count, so it's ok that we're in a synch block
-        return core;
-      }
+      return transientCores.get(name);
     }
-
-    return null;
-
   }
 
   protected CoreDescriptor getDynamicDescriptor(String name) {
@@ -1719,29 +1721,22 @@ class CoreMaps {
       }
     }
   }
-  // We get here when we're being loaded, and the presumption is that we're not in the list yet.
-  protected SolrCore waitPendingCoreOps(String name) {
-
-    // Keep multiple threads from opening or closing a core at one time.
-    SolrCore ret = null;
+  // Wait here until any pending operations (load, unload or reload) are completed on this core.
+  protected SolrCore waitAddPendingCoreOps(String name) {
 
+    // Keep multiple threads from operating on a core at one time.
     synchronized (locker) {
       boolean pending;
-      do { // We're either loading or unloading this core,
-        pending = pendingDynamicLoads.contains(name); // wait for the core to be loaded
-        if (! pending) {
-          // Check pending closes. This is a linear search is inefficient, but maps don't work without a lot of complexity,
-          // we'll live with it unless it proves to be a bottleneck. In the "usual" case, this list shouldn't be
-          // very long. In the stress test associated with SOLR-4196, this hovered around 0-3, occasionally spiking
-          // very briefly to around 30.
-          for (SolrCore core : pendingDynamicCloses) {
+      do { // Are we currently doing anything to this core? Loading, unloading, reloading?
+        pending = pendingCoreOps.contains(name); // wait for the core to be done being operated upon
+        if (! pending) { // Linear list, but shouldn't be too long
+          for (SolrCore core : pendingCloses) {
             if (core.getName().equals(name)) {
               pending = true;
               break;
             }
           }
         }
-
         if (container.isShutDown()) return null; // Just stop already.
 
         if (pending) {
@@ -1752,26 +1747,29 @@ class CoreMaps {
           }
         }
       } while (pending);
-
-      if (!container.isShutDown()) {
-        ret = getCoreFromAnyList(name); // we might have been _unloading_ the core, so check.
-        if (ret == null) {
-          pendingDynamicLoads.add(name); // the caller is going to load us. If we happen to be shutting down, we don't care.
+      // We _really_ need to do this within the synchronized block!
+      if (! container.isShutDown()) {
+        if (! pendingCoreOps.add(name)) {
+          CoreContainer.log.warn("Replaced an entry in pendingCoreOps {}, we should not be doing this", name);
         }
+        return getCoreFromAnyList(name); // we might have been _unloading_ the core, so return the core if it was loaded.
       }
     }
-
-    return ret;
+    return null;
   }
 
-  // The core is loaded, remove it from the pendin gloads
-  protected void releasePending(String name) {
+  // We should always be removing the first thing in the list with our name! The idea here is to NOT do anything n
+  // any core while some other operation is working on that core.
+  protected void removeFromPendingOps(String name) {
     synchronized (locker) {
-      pendingDynamicLoads.remove(name);
+      if (! pendingCoreOps.remove(name)) {
+        CoreContainer.log.warn("Tried to remove core {} from pendingCoreOps and it wasn't there. ", name);
+      }
       locker.notifyAll();
     }
   }
 
+
   protected void persistCores(ConfigSolr cfg, Map<String, SolrCore> whichCores, SolrResourceLoader loader) {
     for (SolrCore solrCore : whichCores.values()) {
       addPersistOneCore(cfg, solrCore, loader);
@@ -1880,13 +1878,14 @@ class CoreMaps {
 
   // Be a little careful. We don't want to either open or close a core unless it's _not_ being opened or closed by
   // another thread. So within this lock we'll walk along the list of pending closes until we find something NOT in
-  // the list of threads currently being opened. The "usual" case will probably return the very first one anyway..
+  // the list of threads currently being loaded or reloaded. The "usual" case will probably return the very first
+  // one anyway..
   protected SolrCore getCoreToClose() {
     synchronized (locker) {
-      if (pendingDynamicCloses.size() == 0) return null; // nothing to do.
-      // Yes, a linear search but this is a pretty short list in the normal case and usually we'll take the first one.
-      for (SolrCore core : pendingDynamicCloses) {
-        if (! pendingDynamicLoads.contains(core.getName())) {  // Don't try close a core if it's being opened.
+      for (SolrCore core : pendingCloses) {
+        if (! pendingCoreOps.contains(core.getName())) {
+          pendingCoreOps.add(core.getName());
+          pendingCloses.remove(core);
           return core;
         }
       }
@@ -1894,12 +1893,7 @@ class CoreMaps {
     return null;
   }
 
-  protected void removeClosedFromCloser(SolrCore core) {
-    synchronized (locker) {
-      pendingDynamicCloses.remove(core);
-      locker.notifyAll();
-    }
-  }
+
 }
 
 class CloserThread extends Thread {
@@ -1936,7 +1930,7 @@ class CloserThread extends Thread {
           coreMaps.addPersistOneCore(cfg, removeMe, container.loader);
           removeMe.close();
         } finally {
-          coreMaps.removeClosedFromCloser(removeMe);
+          coreMaps.removeFromPendingOps(removeMe.getName());
         }
       }
     }

Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/CoreDescriptor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/CoreDescriptor.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/CoreDescriptor.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/CoreDescriptor.java Wed Mar 20 16:27:13 2013
@@ -64,6 +64,8 @@ public class CoreDescriptor {
   // them individually.
   private Properties coreProperties = new Properties();
 
+  private boolean loadedImplicit = false;
+
   private final CoreContainer coreContainer;
 
   private CloudDescriptor cloudDesc;
@@ -133,11 +135,11 @@ public class CoreDescriptor {
 
   public Properties initImplicitProperties() {
     Properties implicitProperties = new Properties(coreContainer.getContainerProperties());
-    implicitProperties.setProperty(CORE_NAME, getName());
-    implicitProperties.setProperty(CORE_INSTDIR, getInstanceDir());
-    implicitProperties.setProperty(CORE_DATADIR, getDataDir());
-    implicitProperties.setProperty(CORE_CONFIG, getConfigName());
-    implicitProperties.setProperty(CORE_SCHEMA, getSchemaName());
+    implicitProperties.setProperty("solr.core.name", getName());
+    implicitProperties.setProperty("solr.core.instanceDir", getInstanceDir());
+    implicitProperties.setProperty("solr.core.dataDir", getDataDir());
+    implicitProperties.setProperty("solr.core.configName", getConfigName());
+    implicitProperties.setProperty("solr.core.schemaName", getSchemaName());
     return implicitProperties;
   }
 
@@ -166,19 +168,8 @@ public class CoreDescriptor {
 
   public String getDataDir() {
     String dataDir = coreProperties.getProperty(CORE_DATADIR);
-    if (dataDir == null) {
-      dataDir = getDefaultDataDir();
-    }
-    if (new File(dataDir).isAbsolute()) {
-      return dataDir;
-    } else {
-      if (new File(getInstanceDir()).isAbsolute()) {
-        return SolrResourceLoader.normalizeDir(SolrResourceLoader.normalizeDir(getInstanceDir()) + dataDir);
-      } else  {
-        return SolrResourceLoader.normalizeDir(coreContainer.getSolrHome() +
-                SolrResourceLoader.normalizeDir(getRawInstanceDir()) + dataDir);
-      }
-    }
+    if (dataDir == null) dataDir = getDefaultDataDir();
+    return dataDir;
   }
 
   public void setDataDir(String s) {
@@ -262,13 +253,14 @@ public class CoreDescriptor {
    * Under any circumstance, the properties passed in will override any already present.Merge
    */
   public void setCoreProperties(Properties coreProperties) {
-    if (this.coreProperties == null) {
+    if (! loadedImplicit) {
+      loadedImplicit = true;
       Properties p = initImplicitProperties();
-      this.coreProperties = new Properties(p);
-    }
-    // The caller presumably wants whatever properties passed in to override the current core props, so just add them.
-    if(coreProperties != null) {
-      this.coreProperties.putAll(coreProperties);
+      this.coreProperties.putAll(p);
+      // The caller presumably wants whatever properties passed in to override the current core props, so just add them.
+      if (coreProperties != null) {
+        this.coreProperties.putAll(coreProperties);
+      }
     }
   }
 

Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java Wed Mar 20 16:27:13 2013
@@ -18,6 +18,7 @@ package org.apache.solr.core;
  */
 
 import java.io.Closeable;
+import java.io.File;
 import java.io.IOException;
 
 import org.apache.lucene.store.Directory;
@@ -77,9 +78,10 @@ public abstract class DirectoryFactory i
   
   /**
    * Returns true if a Directory exists for a given path.
+   * @throws IOException If there is a low-level I/O error.
    * 
    */
-  public abstract boolean exists(String path);
+  public abstract boolean exists(String path) throws IOException;
   
   /**
    * Removes the Directory's persistent storage.
@@ -91,6 +93,26 @@ public abstract class DirectoryFactory i
   public abstract void remove(Directory dir) throws IOException;
   
   /**
+   * Removes the Directory's persistent storage.
+   * For example: A file system impl may remove the
+   * on disk directory.
+   * @throws IOException If there is a low-level I/O error.
+   * 
+   */
+  public abstract void remove(Directory dir, boolean afterCoreClose) throws IOException;
+  
+  /**
+   * This remove is special in that it may be called even after
+   * the factory has been closed. Remove only makes sense for
+   * persistent directory factories.
+   * 
+   * @param path to remove
+   * @param afterCoreClose whether to wait until after the core is closed.
+   * @throws IOException If there is a low-level I/O error.
+   */
+  public abstract void remove(String path, boolean afterCoreClose) throws IOException;
+  
+  /**
    * This remove is special in that it may be called even after
    * the factory has been closed. Remove only makes sense for
    * persistent directory factories.
@@ -172,6 +194,15 @@ public abstract class DirectoryFactory i
     return path;
   }
   
+  /**
+   * @param path the path to check
+   * @return true if absolute, as in not relative
+   */
+  public boolean isAbsolute(String path) {
+    // back compat
+    return new File(path).isAbsolute();
+  }
+  
   public static long sizeOfDirectory(Directory directory) throws IOException {
     final String[] files = directory.listAll();
     long size = 0;

Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/EphemeralDirectoryFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/EphemeralDirectoryFactory.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/EphemeralDirectoryFactory.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/EphemeralDirectoryFactory.java Wed Mar 20 16:27:13 2013
@@ -19,7 +19,9 @@ package org.apache.solr.core;
 import java.io.File;
 import java.io.IOException;
 
+import org.apache.commons.io.FileUtils;
 import org.apache.lucene.store.Directory;
+import org.apache.solr.core.CachingDirectoryFactory.CacheValue;
 
 /**
  * Directory provider for implementations that do not persist over reboots.
@@ -28,8 +30,8 @@ import org.apache.lucene.store.Directory
 public abstract class EphemeralDirectoryFactory extends CachingDirectoryFactory {
   
   @Override
-  public boolean exists(String path) {
-    String fullPath = new File(path).getAbsolutePath();
+  public boolean exists(String path) throws IOException {
+    String fullPath = normalize(path);
     synchronized (this) {
       CacheValue cacheValue = byPathCache.get(fullPath);
       Directory directory = null;
@@ -49,17 +51,7 @@ public abstract class EphemeralDirectory
   }
   
   @Override
-  public void remove(Directory dir) throws IOException {
-    // ram dir does not persist its dir anywhere
-  }
-  
-  @Override
-  public void remove(String path) throws IOException {
-    // ram dir does not persist its dir anywhere
-  }
-  
-  @Override
-  public String normalize(String path) throws IOException {
-    return path;
+  public boolean isAbsolute(String path) {
+    return true;
   }
 }

Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/MMapDirectoryFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/MMapDirectoryFactory.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/MMapDirectoryFactory.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/MMapDirectoryFactory.java Wed Mar 20 16:27:13 2013
@@ -66,4 +66,9 @@ public class MMapDirectoryFactory extend
     }
     return mapDirectory;
   }
+  
+  @Override
+  public boolean isAbsolute(String path) {
+    return new File(path).isAbsolute();
+  }
 }

Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/NIOFSDirectoryFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/NIOFSDirectoryFactory.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/NIOFSDirectoryFactory.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/NIOFSDirectoryFactory.java Wed Mar 20 16:27:13 2013
@@ -16,13 +16,12 @@ package org.apache.solr.core;
  * limitations under the License.
  */
 
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.NIOFSDirectory;
-import org.apache.solr.core.DirectoryFactory.DirContext;
-
 import java.io.File;
 import java.io.IOException;
 
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.NIOFSDirectory;
+
 
 /**
  * Factory to instantiate {@link org.apache.lucene.store.NIOFSDirectory}
@@ -35,4 +34,9 @@ public class NIOFSDirectoryFactory exten
     return new NIOFSDirectory(new File(path));
   }
   
+  @Override
+  public boolean isAbsolute(String path) {
+    return new File(path).isAbsolute();
+  }
+  
 }

Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/NRTCachingDirectoryFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/NRTCachingDirectoryFactory.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/NRTCachingDirectoryFactory.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/NRTCachingDirectoryFactory.java Wed Mar 20 16:27:13 2013
@@ -25,7 +25,6 @@ import org.apache.lucene.store.FSDirecto
 import org.apache.lucene.store.NRTCachingDirectory;
 import org.apache.solr.common.params.SolrParams;
 import org.apache.solr.common.util.NamedList;
-import org.apache.solr.core.DirectoryFactory.DirContext;
 
 /**
  * Factory to instantiate {@link org.apache.lucene.store.NRTCachingDirectory}
@@ -54,5 +53,10 @@ public class NRTCachingDirectoryFactory 
   protected Directory create(String path, DirContext dirContext) throws IOException {
     return new NRTCachingDirectory(FSDirectory.open(new File(path)), maxMergeSizeMB, maxCachedMB);
   }
+  
+  @Override
+  public boolean isAbsolute(String path) {
+    return new File(path).isAbsolute();
+  }
 
 }

Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/SimpleFSDirectoryFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/SimpleFSDirectoryFactory.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/SimpleFSDirectoryFactory.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/SimpleFSDirectoryFactory.java Wed Mar 20 16:27:13 2013
@@ -16,13 +16,12 @@ package org.apache.solr.core;
  * limitations under the License.
  */
 
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.SimpleFSDirectory;
-import org.apache.solr.core.DirectoryFactory.DirContext;
-
 import java.io.File;
 import java.io.IOException;
 
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.SimpleFSDirectory;
+
 
 /**
  * Factory to instantiate {@link org.apache.lucene.store.SimpleFSDirectory}
@@ -35,4 +34,8 @@ public class SimpleFSDirectoryFactory ex
     return new SimpleFSDirectory(new File(path));
   }
 
+  @Override
+  public boolean isAbsolute(String path) {
+    return new File(path).isAbsolute();
+  }
 }

Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/SolrCore.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/SolrCore.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/SolrCore.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/SolrCore.java Wed Mar 20 16:27:13 2013
@@ -406,7 +406,7 @@ public final class SolrCore implements S
     }
     
     SolrCore core = new SolrCore(getName(), getDataDir(), config,
-        schema, coreDescriptor, updateHandler, prev);
+        schema, coreDescriptor, updateHandler, this.solrDelPolicy, prev);
     core.solrDelPolicy = this.solrDelPolicy;
     
     core.getUpdateHandler().getSolrCoreState().newIndexWriter(core, false, false);
@@ -459,7 +459,7 @@ public final class SolrCore implements S
       boolean indexExists = getDirectoryFactory().exists(indexDir);
       boolean firstTime;
       synchronized (SolrCore.class) {
-        firstTime = dirs.add(new File(indexDir).getCanonicalPath());
+        firstTime = dirs.add(getDirectoryFactory().normalize(indexDir));
       }
       boolean removeLocks = solrConfig.unlockOnStartup;
 
@@ -616,7 +616,7 @@ public final class SolrCore implements S
    * @since solr 1.3
    */
   public SolrCore(String name, String dataDir, SolrConfig config, IndexSchema schema, CoreDescriptor cd) {
-    this(name, dataDir, config, schema, cd, null, null);
+    this(name, dataDir, config, schema, cd, null, null, null);
   }
 
 
@@ -652,16 +652,28 @@ public final class SolrCore implements S
    *
    *@since solr 1.3
    */
-  public SolrCore(String name, String dataDir, SolrConfig config, IndexSchema schema, CoreDescriptor cd, UpdateHandler updateHandler, SolrCore prev) {
+  public SolrCore(String name, String dataDir, SolrConfig config, IndexSchema schema, CoreDescriptor cd, UpdateHandler updateHandler, IndexDeletionPolicyWrapper delPolicy, SolrCore prev) {
     coreDescriptor = cd;
     this.setName( name );
     resourceLoader = config.getResourceLoader();
-    if (dataDir == null){
-      if(cd.usingDefaultDataDir()) {
-        dataDir = config.getDataDir();
-      }
-      if(dataDir == null) {
+    this.solrConfig = config;
+    
+    if (updateHandler == null) {
+      initDirectoryFactory();
+    }
+    
+    if (dataDir == null) {
+      if (cd.usingDefaultDataDir()) dataDir = config.getDataDir();
+      if (dataDir == null) {
         dataDir = cd.getDataDir();
+        try {
+          if (!directoryFactory.isAbsolute(dataDir)) {
+            dataDir = directoryFactory.normalize(SolrResourceLoader
+                .normalizeDir(cd.getInstanceDir()) + dataDir);
+          }
+        } catch (IOException e) {
+          throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, null, e);
+        }
       }
     }
 
@@ -703,7 +715,6 @@ public final class SolrCore implements S
 
     this.schema = schema;
     this.dataDir = dataDir;
-    this.solrConfig = config;
     this.startTime = System.currentTimeMillis();
     this.maxWarmingSearchers = config.maxWarmingSearchers;
 
@@ -715,8 +726,10 @@ public final class SolrCore implements S
       
       initListeners();
       
-      if (updateHandler == null) {
+      if (delPolicy == null) {
         initDeletionPolicy();
+      } else {
+        this.solrDelPolicy = delPolicy;
       }
       
       this.codec = initCodec(solrConfig, schema);

Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/SolrProperties.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/SolrProperties.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/SolrProperties.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/SolrProperties.java Wed Mar 20 16:27:13 2013
@@ -27,6 +27,7 @@ import org.apache.solr.handler.component
 import org.apache.solr.schema.IndexSchema;
 import org.apache.solr.util.PropertiesUtil;
 import org.apache.solr.util.SystemIdResolver;
+import org.apache.solr.util.plugin.PluginInfoInitialized;
 import org.apache.zookeeper.KeeperException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -104,13 +105,15 @@ public class SolrProperties implements C
    * Create a SolrProperties object from an opened input stream, useful for creating defaults
    *
    * @param container - the container for this Solr instance. There should be one and only one...
+   * @param loader    - Solr resource loader
    * @param is        - Input stream for loading properties.
    * @param fileName  - the name for this properties object.
    * @throws IOException - It's possible to walk a very deep tree, if that process goes awry, or if reading any
    *                     of the files found doesn't work, you'll get an IO exception
    */
-  public SolrProperties(CoreContainer container, InputStream is, String fileName) throws IOException {
+  public SolrProperties(CoreContainer container, SolrResourceLoader loader, InputStream is, String fileName) throws IOException {
     origsolrprops.load(is);
+    this.loader = loader;
     this.container = container;
     init(fileName);
   }
@@ -241,29 +244,38 @@ public class SolrProperties implements C
     boolean haveHandler = false;
     for (String s : solrProperties.stringPropertyNames()) {
       String val = solrProperties.getProperty(s);
-      if (s.indexOf(SHARD_HANDLER_FACTORY) != -1) {
+      int index = s.indexOf(SHARD_HANDLER_FACTORY);
+      if (index != -1) {
         haveHandler = true;
         if (SHARD_HANDLER_NAME.equals(s) || SHARD_HANDLER_CLASS.equals(s)) {
-          attrs.put(s, val);
+          // remove shardHandlerFactory. prefix
+          attrs.put(s.substring(SHARD_HANDLER_FACTORY.length()+1), val);
         } else {
-          args.add(s, val);
+          // remove shardHandlerFactory. prefix
+          args.add(s.substring(SHARD_HANDLER_FACTORY.length()+1), val);
         }
       }
     }
 
     if (haveHandler) {
-      //  public PluginInfo(String type, Map<String, String> attrs ,NamedList initArgs, List<PluginInfo> children) {
-
       info = new PluginInfo(SHARD_HANDLER_FACTORY, attrs, args, null);
     } else {
       Map m = new HashMap();
       m.put("class", HttpShardHandlerFactory.class.getName());
       info = new PluginInfo("shardHandlerFactory", m, null, Collections.<PluginInfo>emptyList());
     }
-    HttpShardHandlerFactory fac = new HttpShardHandlerFactory();
-    if (info != null) {
-      fac.init(info);
+
+    assert loader != null;
+    ShardHandlerFactory fac;
+    try {
+      fac = loader.findClass(info.className, ShardHandlerFactory.class).newInstance();
+    } catch (Exception e) {
+      throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
+    }
+    if (fac instanceof PluginInfoInitialized) {
+      ((PluginInfoInitialized) fac).init(info);
     }
+
     return fac;
   }
 

Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java Wed Mar 20 16:27:13 2013
@@ -25,6 +25,7 @@ import org.apache.lucene.store.FSDirecto
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.NRTCachingDirectory;
 import org.apache.lucene.store.RateLimitedDirectoryWrapper;
+import org.apache.solr.core.CachingDirectoryFactory.CacheValue;
 
 /**
  * Directory provider which mimics original Solr 
@@ -43,7 +44,9 @@ public class StandardDirectoryFactory ex
   
   @Override
   public String normalize(String path) throws IOException {
-    return new File(path).getCanonicalPath();
+    String cpath = new File(path).getCanonicalPath();
+    
+    return super.normalize(cpath);
   }
   
   public boolean isPersistent() {
@@ -51,20 +54,14 @@ public class StandardDirectoryFactory ex
   }
   
   @Override
-  public void remove(Directory dir) throws IOException {
-    CacheValue val = byDirectoryCache.get(dir);
-    if (val == null) {
-      throw new IllegalArgumentException("Unknown directory " + dir);
-    }
-    File dirFile = new File(val.path);
-    FileUtils.deleteDirectory(dirFile);
+  public boolean isAbsolute(String path) {
+    // back compat
+    return new File(path).isAbsolute();
   }
   
-
   @Override
-  public void remove(String path) throws IOException {
-    String fullPath = new File(path).getAbsolutePath();
-    File dirFile = new File(fullPath);
+  protected void removeDirectory(CacheValue cacheValue) throws IOException {
+    File dirFile = new File(cacheValue.path);
     FileUtils.deleteDirectory(dirFile);
   }
   

Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/SnapPuller.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/SnapPuller.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/SnapPuller.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/SnapPuller.java Wed Mar 20 16:27:13 2013
@@ -439,8 +439,12 @@ public class SnapPuller {
             @Override
             public void preClose() {
               LOG.info("removing old index files " + freezeIndexDir);
-              if (core.getDirectoryFactory().exists(freezeIndexDirPath)) {
-                DirectoryFactory.empty(freezeIndexDir);
+              try {
+                if (core.getDirectoryFactory().exists(freezeIndexDirPath)) {
+                  DirectoryFactory.empty(freezeIndexDir);
+                }
+              } catch (IOException e) {
+                SolrException.log(LOG, null, e);
               }
             }
             
@@ -477,13 +481,6 @@ public class SnapPuller {
         throw new InterruptedException("Index fetch interrupted");
       } catch (Exception e) {
         throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Index fetch failed : ", e);
-      } finally {
-        if (deleteTmpIdxDir) {
-          LOG.info("removing temporary index download directory files " + tmpIndexDir);
-          if (tmpIndex != null && core.getDirectoryFactory().exists(tmpIndex)) {
-            DirectoryFactory.empty(tmpIndexDir);
-          }
-        } 
       }
     } finally {
       try {
@@ -500,9 +497,6 @@ public class SnapPuller {
         stop = false;
         fsyncException = null;
       } finally {
-        if (tmpIndexDir != null) {
-          core.getDirectoryFactory().release(tmpIndexDir);
-        }
         if (deleteTmpIdxDir && tmpIndexDir != null) {
           try {
             core.getDirectoryFactory().remove(tmpIndexDir);
@@ -510,6 +504,11 @@ public class SnapPuller {
             SolrException.log(LOG, "Error removing directory " + tmpIndexDir, e);
           }
         }
+        
+        if (tmpIndexDir != null) {
+          core.getDirectoryFactory().release(tmpIndexDir);
+        }
+        
         if (indexDir != null) {
           core.getDirectoryFactory().release(indexDir);
         }
@@ -674,13 +673,14 @@ public class SnapPuller {
     
   }
 
-
   /**
    * All the files are copied to a temp dir first
    */
   private String createTempindexDir(SolrCore core, String tmpIdxDirName) {
-    File tmpIdxDir = new File(core.getDataDir(), tmpIdxDirName);
-    return tmpIdxDir.toString();
+    // TODO: there should probably be a DirectoryFactory#concatPath(parent, name)
+    // or something
+    String tmpIdxDir = core.getDataDir() + tmpIdxDirName;
+    return tmpIdxDir;
   }
 
   private void reloadCore() {

Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java Wed Mar 20 16:27:13 2013
@@ -40,6 +40,7 @@ import org.apache.solr.common.params.Cor
 import org.apache.solr.common.params.ModifiableSolrParams;
 import org.apache.solr.common.params.SolrParams;
 import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SimpleOrderedMap;
 import org.apache.solr.core.CoreContainer;
 import org.apache.solr.handler.RequestHandlerBase;
 import org.apache.solr.request.SolrQueryRequest;
@@ -102,35 +103,38 @@ public class CollectionsHandler extends 
     if (a != null) {
       action = CollectionAction.get(a);
     }
-    if (action != null) {
-      switch (action) {
-        case CREATE: {
-          this.handleCreateAction(req, rsp);
-          break;
-        }
-        case DELETE: {
-          this.handleDeleteAction(req, rsp);
-          break;
-        }
-        case RELOAD: {
-          this.handleReloadAction(req, rsp);
-          break;
-        }
-        case SYNCSHARD: {
-          this.handleSyncShardAction(req, rsp);
-          break;
-        }
-        case CREATEALIAS: {
-          this.handleCreateAliasAction(req, rsp);
-          break;
-        }
-        case DELETEALIAS: {
-          this.handleDeleteAliasAction(req, rsp);
-          break;
-        }
-        default: {
-          throw new RuntimeException("Unknown action: " + action);
-        }
+    if (action == null) {
+      throw new SolrException(ErrorCode.BAD_REQUEST, "Unknown action: " + a);
+    }
+    
+    switch (action) {
+      case CREATE: {
+        this.handleCreateAction(req, rsp);
+        break;
+      }
+      case DELETE: {
+        this.handleDeleteAction(req, rsp);
+        break;
+      }
+      case RELOAD: {
+        this.handleReloadAction(req, rsp);
+        break;
+      }
+      case SYNCSHARD: {
+        this.handleSyncShardAction(req, rsp);
+        break;
+      }
+      case CREATEALIAS: {
+        this.handleCreateAliasAction(req, rsp);
+        break;
+      }
+      case DELETEALIAS: {
+        this.handleDeleteAliasAction(req, rsp);
+        break;
+      }
+      default: {
+        throw new SolrException(ErrorCode.BAD_REQUEST, "Unknown action: "
+            + action);
       }
     }
 
@@ -148,6 +152,11 @@ public class CollectionsHandler extends 
     if (event.getBytes() != null) {
       SolrResponse response = SolrResponse.deserialize(event.getBytes());
       rsp.getValues().addAll(response.getResponse());
+      SimpleOrderedMap exp = (SimpleOrderedMap) response.getResponse().get("exception");
+      if (exp != null) {
+        Integer code = (Integer) exp.get("rspCode");
+        rsp.setException(new SolrException(code != null && code != -1 ? ErrorCode.getErrorCode(code) : ErrorCode.SERVER_ERROR, (String)exp.get("msg")));
+      }
     } else {
       if (System.currentTimeMillis() - time >= DEFAULT_ZK_TIMEOUT) {
         throw new SolrException(ErrorCode.SERVER_ERROR, operation

Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java Wed Mar 20 16:27:13 2013
@@ -29,8 +29,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 
-import javax.xml.parsers.ParserConfigurationException;
-
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang.StringUtils;
 import org.apache.lucene.index.DirectoryReader;
@@ -73,7 +71,6 @@ import org.apache.solr.util.RefCounted;
 import org.apache.zookeeper.KeeperException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.xml.sax.SAXException;
 
 /**
  *
@@ -408,8 +405,8 @@ public class CoreAdminHandler extends Re
       //for now, do not allow creating new core with same name when in cloud mode
       //XXX perhaps it should just be unregistered from cloud before readding it?, 
       //XXX perhaps we should also check that cores are of same type before adding new core to collection?
-      if (coreContainer.getZkController() != null) {
-        if (coreContainer.getCore(name) != null) {
+      if (coreContainer.isZooKeeperAware()) {
+        if (coreContainer.getCoreNames().contains(name)) {
           log.info("Re-creating a core with existing name is not allowed in cloud mode");
           throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
               "Core with name '" + name + "' already exists.");
@@ -568,43 +565,24 @@ public class CoreAdminHandler extends Re
                     + e.getMessage(), e);
           }
         }
-      }
-      if (params.getBool(CoreAdminParams.DELETE_INDEX, false)) {
-        core.addCloseHook(new CloseHook() {
-          private String indexDir;
-          
-          @Override
-          public void preClose(SolrCore core) {
-            indexDir = core.getIndexDir();
-          }
-          
-          @Override
-          public void postClose(SolrCore core) {
-            try {
-              core.getDirectoryFactory().remove(indexDir);
-            } catch (IOException e) {
-              throw new RuntimeException(e);
-            }
+        
+        if (params.getBool(CoreAdminParams.DELETE_INDEX, false)) {
+          try {
+            core.getDirectoryFactory().remove(core.getIndexDir());
+          } catch (Exception e) {
+            SolrException.log(log, "Failed to flag index dir for removal for core:"
+                    + core.getName() + " dir:" + core.getIndexDir());
           }
-        });
+        }
       }
-      
+
       if (params.getBool(CoreAdminParams.DELETE_DATA_DIR, false)) {
-        core.addCloseHook(new CloseHook() {
-          @Override
-          public void preClose(SolrCore core) {}
-          
-          @Override
-          public void postClose(SolrCore core) {
-            File dataDir = new File(core.getDataDir());
-            try {
-              FileUtils.deleteDirectory(dataDir);
-            } catch (IOException e) {
-              SolrException.log(log, "Failed to delete data dir for core:"
-                  + core.getName() + " dir:" + dataDir.getAbsolutePath());
-            }
-          }
-        });
+        try {
+          core.getDirectoryFactory().remove(core.getDataDir(), true);
+        } catch (Exception e) {
+          SolrException.log(log, "Failed to flag data dir for removal for core:"
+                  + core.getName() + " dir:" + core.getDataDir());
+        }
       }
       
       if (params.getBool(CoreAdminParams.DELETE_INSTANCE_DIR, false)) {

Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java Wed Mar 20 16:27:13 2013
@@ -18,7 +18,6 @@ package org.apache.solr.handler.componen
 
 import java.net.ConnectException;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -27,7 +26,6 @@ import java.util.Set;
 import java.util.concurrent.Callable;
 import java.util.concurrent.CompletionService;
 import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorCompletionService;
 import java.util.concurrent.Future;
 
 import org.apache.http.client.HttpClient;
@@ -61,21 +59,20 @@ public class HttpShardHandler extends Sh
 
   private HttpShardHandlerFactory httpShardHandlerFactory;
   private CompletionService<ShardResponse> completionService;
-  private     Set<Future<ShardResponse>> pending;
+  private Set<Future<ShardResponse>> pending;
   private Map<String,List<String>> shardToURLs;
   private HttpClient httpClient;
 
 
-
   public HttpShardHandler(HttpShardHandlerFactory httpShardHandlerFactory, HttpClient httpClient) {
     this.httpClient = httpClient;
     this.httpShardHandlerFactory = httpShardHandlerFactory;
-    completionService = new ExecutorCompletionService<ShardResponse>(httpShardHandlerFactory.commExecutor);
+    completionService = httpShardHandlerFactory.newCompletionService();
     pending = new HashSet<Future<ShardResponse>>();
 
     // maps "localhost:8983|localhost:7574" to a shuffled List("http://localhost:8983","http://localhost:7574")
-      // This is primarily to keep track of what order we should use to query the replicas of a shard
-      // so that we use the same replica for all phases of a distributed request.
+    // This is primarily to keep track of what order we should use to query the replicas of a shard
+    // so that we use the same replica for all phases of a distributed request.
     shardToURLs = new HashMap<String,List<String>>();
 
   }
@@ -106,21 +103,8 @@ public class HttpShardHandler extends Sh
   // Don't modify the returned URL list.
   private List<String> getURLs(String shard) {
     List<String> urls = shardToURLs.get(shard);
-    if (urls==null) {
-      urls = StrUtils.splitSmart(shard, "|", true);
-
-      // convert shard to URL
-      for (int i=0; i<urls.size(); i++) {
-        urls.set(i, httpShardHandlerFactory.scheme + urls.get(i));
-      }
-
-      //
-      // Shuffle the list instead of use round-robin by default.
-      // This prevents accidental synchronization where multiple shards could get in sync
-      // and query the same replica at the same time.
-      //
-      if (urls.size() > 1)
-        Collections.shuffle(urls, httpShardHandlerFactory.r);
+    if (urls == null) {
+      urls = httpShardHandlerFactory.makeURLList(shard);
       shardToURLs.put(shard, urls);
     }
     return urls;
@@ -137,6 +121,9 @@ public class HttpShardHandler extends Sh
       public ShardResponse call() throws Exception {
 
         ShardResponse srsp = new ShardResponse();
+        if (sreq.nodeName != null) {
+          srsp.setNodeName(sreq.nodeName);
+        }
         srsp.setShardRequest(sreq);
         srsp.setShard(shard);
         SimpleSolrResponse ssr = new SimpleSolrResponse();
@@ -168,7 +155,7 @@ public class HttpShardHandler extends Sh
             SolrServer server = new HttpSolrServer(url, httpClient);
             ssr.nl = server.request(req);
           } else {
-            LBHttpSolrServer.Rsp rsp = httpShardHandlerFactory.loadbalancer.request(new LBHttpSolrServer.Req(req, urls));
+            LBHttpSolrServer.Rsp rsp = httpShardHandlerFactory.makeLoadBalancedRequest(req, urls);
             ssr.nl = rsp.getResponse();
             srsp.setShardAddress(rsp.getServer());
           }

Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java Wed Mar 20 16:27:13 2013
@@ -16,24 +16,36 @@ package org.apache.solr.handler.componen
  * limitations under the License.
  */
 
+import java.io.IOException;
 import java.net.MalformedURLException;
+import java.util.Collections;
+import java.util.List;
 import java.util.Random;
-import java.util.concurrent.*;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
 
 import org.apache.http.client.HttpClient;
+import org.apache.solr.client.solrj.SolrServerException;
 import org.apache.solr.client.solrj.impl.HttpClientUtil;
 import org.apache.solr.client.solrj.impl.LBHttpSolrServer;
+import org.apache.solr.client.solrj.request.QueryRequest;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.params.ModifiableSolrParams;
 import org.apache.solr.common.util.ExecutorUtil;
 import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.StrUtils;
 import org.apache.solr.core.PluginInfo;
 import org.apache.solr.util.DefaultSolrThreadFactory;
-import org.apache.solr.util.plugin.PluginInfoInitialized;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-public class HttpShardHandlerFactory extends ShardHandlerFactory implements PluginInfoInitialized {
+
+public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.apache.solr.util.plugin.PluginInfoInitialized {
   protected static Logger log = LoggerFactory.getLogger(HttpShardHandlerFactory.class);
 
   // We want an executor that doesn't take up any resources if
@@ -42,7 +54,7 @@ public class HttpShardHandlerFactory ext
   //
   // Consider CallerRuns policy and a lower max threads to throttle
   // requests at some point (or should we simply return failure?)
-  ThreadPoolExecutor commExecutor = new ThreadPoolExecutor(
+  private ThreadPoolExecutor commExecutor = new ThreadPoolExecutor(
       0,
       Integer.MAX_VALUE,
       5, TimeUnit.SECONDS, // terminate idle threads after 5 sec
@@ -51,7 +63,7 @@ public class HttpShardHandlerFactory ext
   );
 
   private HttpClient defaultClient;
-  LBHttpSolrServer loadbalancer;
+  private LBHttpSolrServer loadbalancer;
   //default values:
   int soTimeout = 0; 
   int connectionTimeout = 0; 
@@ -62,9 +74,9 @@ public class HttpShardHandlerFactory ext
   int queueSize = -1;
   boolean accessPolicy = false;
 
-  public String scheme = "http://"; //current default values
+  private String scheme = "http://"; //current default values
 
-  final Random r = new Random();
+  private final Random r = new Random();
 
   // URL scheme to be used in distributed search.
   static final String INIT_URL_SCHEME = "urlScheme";
@@ -112,6 +124,12 @@ public class HttpShardHandlerFactory ext
     this.keepAliveTime = getParameter(args, MAX_THREAD_IDLE_TIME, keepAliveTime);
     this.queueSize = getParameter(args, INIT_SIZE_OF_QUEUE, queueSize);
     this.accessPolicy = getParameter(args, INIT_FAIRNESS_POLICY, accessPolicy);
+    
+    // magic sysprop to make tests reproducible: set by SolrTestCaseJ4.
+    String v = System.getProperty("tests.shardhandler.randomSeed");
+    if (v != null) {
+      r.setSeed(Long.parseLong(v));
+    }
 
     BlockingQueue<Runnable> blockingQueue = (this.queueSize == -1) ?
         new SynchronousQueue<Runnable>(this.accessPolicy) :
@@ -177,4 +195,48 @@ public class HttpShardHandlerFactory ext
     }
 
   }
+
+  /**
+   * Makes a request to one or more of the given urls, using the configured load balancer.
+   *
+   * @param req The solr search request that should be sent through the load balancer
+   * @param urls The list of solr server urls to load balance across
+   * @return The response from the request
+   */
+  public LBHttpSolrServer.Rsp makeLoadBalancedRequest(final QueryRequest req, List<String> urls)
+    throws SolrServerException, IOException {
+    return loadbalancer.request(new LBHttpSolrServer.Req(req, urls));
+  }
+
+  /**
+   * Creates a randomized list of urls for the given shard.
+   *
+   * @param shard the urls for the shard (minus "http://"), separated by '|'
+   * @return A list of valid urls (including protocol) that are replicas for the shard
+   */
+  public List<String> makeURLList(String shard) {
+    List<String> urls = StrUtils.splitSmart(shard, "|", true);
+
+    // convert shard to URL
+    for (int i=0; i<urls.size(); i++) {
+      urls.set(i, scheme + urls.get(i));
+    }
+
+    //
+    // Shuffle the list instead of use round-robin by default.
+    // This prevents accidental synchronization where multiple shards could get in sync
+    // and query the same replica at the same time.
+    //
+    if (urls.size() > 1)
+      Collections.shuffle(urls, r);
+
+    return urls;
+  }
+
+  /**
+   * Creates a new completion service for use by a single set of distributed requests.
+   */
+  public CompletionService newCompletionService() {
+    return new ExecutorCompletionService<ShardResponse>(commExecutor);
+  }
 }

Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.java Wed Mar 20 16:27:13 2013
@@ -53,6 +53,9 @@ public class ShardRequest {
   /** actual shards to send the request to, filled out by framework */
   public String[] actualShards;
 
+  /** may be null */
+  public String nodeName;
+
   // TODO: one could store a list of numbers to correlate where returned docs
   // go in the top-level response rather than looking up by id...
   // this would work well if we ever transitioned to using internal ids and

Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/component/ShardResponse.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/component/ShardResponse.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/component/ShardResponse.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/component/ShardResponse.java Wed Mar 20 16:27:13 2013
@@ -22,6 +22,7 @@ import org.apache.solr.common.SolrExcept
 public final class ShardResponse {
   private ShardRequest req;
   private String shard;
+  private String nodeName;
   private String shardAddress;  // the specific shard that this response was received from
   private int rspCode;
   private Throwable exception;
@@ -56,6 +57,11 @@ public final class ShardResponse {
     return shard;
   }
 
+  public String getNodeName()
+  {
+    return nodeName;
+  }
+  
   public void setShardRequest(ShardRequest rsp)
   {
     this.req = rsp;
@@ -80,9 +86,15 @@ public final class ShardResponse {
   {
     this.rspCode = rspCode;
   }
+  
+  void setNodeName(String nodeName) 
+  {
+    this.nodeName = nodeName;
+  }
 
   /** What was the shard address that returned this response.  Example:  "http://localhost:8983/solr" */
   public String getShardAddress() { return this.shardAddress; }
 
   void setShardAddress(String addr) { this.shardAddress = addr; }
+
 }

Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java Wed Mar 20 16:27:13 2013
@@ -22,8 +22,8 @@ import java.io.StringReader;
 import java.util.*;
 
 import org.apache.commons.io.IOUtils;
-import org.apache.noggit.JSONParser;
-import org.apache.noggit.ObjectBuilder;
+import org.noggit.JSONParser;
+import org.noggit.ObjectBuilder;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.SolrInputDocument;
 import org.apache.solr.common.SolrInputField;

Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/loader/XMLLoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/loader/XMLLoader.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/loader/XMLLoader.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/loader/XMLLoader.java Wed Mar 20 16:27:13 2013
@@ -438,6 +438,7 @@ public class XMLLoader extends ContentSt
           }
           boost = 1.0f;
           update = null;
+          isNull = false;
           String attrVal = "";
           for (int i = 0; i < parser.getAttributeCount(); i++) {
             attrName = parser.getAttributeLocalName(i);

Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/highlight/PostingsSolrHighlighter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/highlight/PostingsSolrHighlighter.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/highlight/PostingsSolrHighlighter.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/highlight/PostingsSolrHighlighter.java Wed Mar 20 16:27:13 2013
@@ -26,8 +26,7 @@ import java.util.Set;
 
 import org.apache.lucene.index.StoredDocument;
 import org.apache.lucene.search.Query;
-import org.apache.lucene.search.ScoreDoc;
-import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.postingshighlight.Passage;
 import org.apache.lucene.search.postingshighlight.PassageFormatter;
 import org.apache.lucene.search.postingshighlight.PassageScorer;
 import org.apache.lucene.search.postingshighlight.PostingsHighlighter;
@@ -55,7 +54,11 @@ import org.apache.solr.util.plugin.Plugi
  *                      preTag="&amp;lt;em&amp;gt;"
  *                      postTag="&amp;lt;/em&amp;gt;"
  *                      ellipsis="... "
- *                      maxLength=10000/&gt;
+ *                      k1="1.2"
+ *                      b="0.75"
+ *                      pivot="87"
+ *                      maxLength=10000
+ *                      summarizeEmpty=true/&gt;
  *   &lt;/searchComponent&gt;
  * </pre>
  * <p>
@@ -78,7 +81,23 @@ public class PostingsSolrHighlighter ext
   public void init(PluginInfo info) {
     Map<String,String> attributes = info.attributes;
     BreakIterator breakIterator = BreakIterator.getSentenceInstance(Locale.ROOT);
-    PassageScorer scorer = new PassageScorer();
+    
+    // scorer parameters: k1/b/pivot
+    String k1 = attributes.get("k1");
+    if (k1 == null) {
+      k1 = "1.2";
+    }
+    
+    String b = attributes.get("b");
+    if (b == null) {
+      b = "0.75";
+    }
+    
+    String pivot = attributes.get("pivot");
+    if (pivot == null) {
+      pivot = "87";
+    }
+    PassageScorer scorer = new PassageScorer(Float.parseFloat(k1), Float.parseFloat(b), Float.parseFloat(pivot));
     
     // formatter parameters: preTag/postTag/ellipsis
     String preTag = attributes.get("preTag");
@@ -94,13 +113,30 @@ public class PostingsSolrHighlighter ext
       ellipsis = "... ";
     }
     PassageFormatter formatter = new PassageFormatter(preTag, postTag, ellipsis);
-    
+
+    String summarizeEmpty = attributes.get("summarizeEmpty");
+    final boolean summarizeEmptyBoolean;
+    if (summarizeEmpty == null) {
+      summarizeEmptyBoolean = true;
+    } else {
+      summarizeEmptyBoolean = Boolean.parseBoolean(summarizeEmpty);
+    }
+
     // maximum content size to process
     int maxLength = PostingsHighlighter.DEFAULT_MAX_LENGTH;
     if (attributes.containsKey("maxLength")) {
       maxLength = Integer.parseInt(attributes.get("maxLength"));
     }
-    highlighter = new PostingsHighlighter(maxLength, breakIterator, scorer, formatter);
+    highlighter = new PostingsHighlighter(maxLength, breakIterator, scorer, formatter) {
+        @Override
+        protected Passage[] getEmptyHighlight(String fieldName, BreakIterator bi, int maxPassages) {
+          if (summarizeEmptyBoolean) {
+            return super.getEmptyHighlight(fieldName, bi, maxPassages);
+          } else {
+            return new Passage[0];
+          }
+        }
+      };
   }
 
   @Override
@@ -110,16 +146,16 @@ public class PostingsSolrHighlighter ext
     // if highlighting isnt enabled, then why call doHighlighting?
     if (isHighlightingEnabled(params)) {
       SolrIndexSearcher searcher = req.getSearcher();
-      TopDocs topDocs = toTopDocs(docs);
+      int[] docIDs = toDocIDs(docs);
       
       // fetch the unique keys
-      String[] keys = getUniqueKeys(searcher, topDocs);
+      String[] keys = getUniqueKeys(searcher, docIDs);
       
       // query-time parameters
       String[] fieldNames = getHighlightFields(query, req, defaultFields);
       int numSnippets = params.getInt(HighlightParams.SNIPPETS, 1);
       
-      Map<String,String[]> snippets = highlighter.highlightFields(fieldNames, query, searcher, topDocs, numSnippets);
+      Map<String,String[]> snippets = highlighter.highlightFields(fieldNames, query, searcher, docIDs, numSnippets);
       return encodeSnippets(keys, fieldNames, snippets);
     } else {
       return null;
@@ -152,38 +188,38 @@ public class PostingsSolrHighlighter ext
     return list;
   }
   
-  /** Converts solr's DocList to a lucene TopDocs */
-  protected TopDocs toTopDocs(DocList docs) {
-    ScoreDoc[] scoreDocs = new ScoreDoc[docs.size()];
+  /** Converts solr's DocList to the int[] docIDs */
+  protected int[] toDocIDs(DocList docs) {
+    int[] docIDs = new int[docs.size()];
     DocIterator iterator = docs.iterator();
-    for (int i = 0; i < scoreDocs.length; i++) {
+    for (int i = 0; i < docIDs.length; i++) {
       if (!iterator.hasNext()) {
         throw new AssertionError();
       }
-      scoreDocs[i] = new ScoreDoc(iterator.nextDoc(), Float.NaN);
+      docIDs[i] = iterator.nextDoc();
     }
     if (iterator.hasNext()) {
       throw new AssertionError();
     }
-    return new TopDocs(docs.matches(), scoreDocs, Float.NaN);
+    return docIDs;
   }
   
   /** Retrieves the unique keys for the topdocs to key the results */
-  protected String[] getUniqueKeys(SolrIndexSearcher searcher, TopDocs topDocs) throws IOException {
+  protected String[] getUniqueKeys(SolrIndexSearcher searcher, int[] docIDs) throws IOException {
     IndexSchema schema = searcher.getSchema();
     SchemaField keyField = schema.getUniqueKeyField();
     if (keyField != null) {
       Set<String> selector = Collections.singleton(keyField.getName());
-      String uniqueKeys[] = new String[topDocs.scoreDocs.length];
-      for (int i = 0; i < topDocs.scoreDocs.length; i++) {
-        int docid = topDocs.scoreDocs[i].doc;
+      String uniqueKeys[] = new String[docIDs.length];
+      for (int i = 0; i < docIDs.length; i++) {
+        int docid = docIDs[i];
         StoredDocument doc = searcher.doc(docid, selector);
         String id = schema.printableUniqueKey(doc);
         uniqueKeys[i] = id;
       }
       return uniqueKeys;
     } else {
-      return new String[topDocs.scoreDocs.length];
+      return new String[docIDs.length];
     }
   }
 }

Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/parser/SolrQueryParserBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/parser/SolrQueryParserBase.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/parser/SolrQueryParserBase.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/parser/SolrQueryParserBase.java Wed Mar 20 16:27:13 2013
@@ -922,7 +922,8 @@ public abstract class SolrQueryParserBas
   }
 
   protected String analyzeIfMultitermTermText(String field, String part, FieldType fieldType) {
-    if (part == null) return part;
+
+    if (part == null || ! (fieldType instanceof TextField) || ((TextField)fieldType).getMultiTermAnalyzer() == null) return part;
 
     SchemaField sf = schema.getFieldOrNull((field));
     if (sf == null || ! (fieldType instanceof TextField)) return part;

Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/request/PerSegmentSingleValuedFaceting.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/request/PerSegmentSingleValuedFaceting.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/request/PerSegmentSingleValuedFaceting.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/request/PerSegmentSingleValuedFaceting.java Wed Mar 20 16:27:13 2013
@@ -23,7 +23,6 @@ import java.util.concurrent.*;
 
 import org.apache.lucene.index.AtomicReaderContext;
 import org.apache.lucene.index.SortedDocValues;
-import org.apache.lucene.index.SortedDocValuesTermsEnum;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.search.DocIdSet;
 import org.apache.lucene.search.DocIdSetIterator;
@@ -155,7 +154,7 @@ class PerSegmentSingleValuedFaceting {
           seg.pos = seg.startTermIndex;
         }
         if (seg.pos < seg.endTermIndex) {
-          seg.tenum = new SortedDocValuesTermsEnum(seg.si);
+          seg.tenum = seg.si.termsEnum();
           seg.tenum.seekExact(seg.pos);
           seg.tempBR = seg.tenum.term();
           queue.add(seg);



Mime
View raw message