lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From markrmil...@apache.org
Subject svn commit: r1498702 [1/2] - in /lucene/dev/branches/branch_4x: ./ lucene/ lucene/tools/ lucene/tools/junit4/ solr/ solr/core/ solr/core/src/java/org/apache/solr/ solr/core/src/java/org/apache/solr/client/solrj/embedded/ solr/core/src/java/org/apache/s...
Date Mon, 01 Jul 2013 21:31:21 GMT
Author: markrmiller
Date: Mon Jul  1 21:31:19 2013
New Revision: 1498702

URL: http://svn.apache.org/r1498702
Log:
SOLR-4916: Add support to write and read Solr index files and transaction log files to and from HDFS.

Added:
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/Assign.java
      - copied unchanged from r1497072, lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/Assign.java
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java
      - copied unchanged from r1497072, lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/
      - copied from r1497072, lucene/dev/trunk/solr/core/src/java/org/apache/solr/store/
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/HdfsTransactionLog.java
      - copied unchanged from r1497072, lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/HdfsTransactionLog.java
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/HdfsUpdateLog.java
      - copied unchanged from r1497072, lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/HdfsUpdateLog.java
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/util/HdfsUtil.java
      - copied unchanged from r1497072, lucene/dev/trunk/solr/core/src/java/org/apache/solr/util/HdfsUtil.java
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/util/IOUtils.java
      - copied unchanged from r1497072, lucene/dev/trunk/solr/core/src/java/org/apache/solr/util/IOUtils.java
    lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/AssignTest.java
      - copied unchanged from r1497072, lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/AssignTest.java
    lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/hdfs/
      - copied from r1497072, lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/hdfs/
    lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java
      - copied unchanged from r1497072, lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java
    lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/store/
      - copied from r1497072, lucene/dev/trunk/solr/core/src/test/org/apache/solr/store/
    lucene/dev/branches/branch_4x/solr/example/alt-configs/
      - copied from r1497072, lucene/dev/trunk/solr/example/alt-configs/
    lucene/dev/branches/branch_4x/solr/licenses/commons-configuration-1.6.jar.sha1
      - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/commons-configuration-1.6.jar.sha1
    lucene/dev/branches/branch_4x/solr/licenses/commons-configuration-LICENSE-ASL.txt
      - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/commons-configuration-LICENSE-ASL.txt
    lucene/dev/branches/branch_4x/solr/licenses/commons-configuration-NOTICE.txt
      - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/commons-configuration-NOTICE.txt
    lucene/dev/branches/branch_4x/solr/licenses/concurrentlinkedhashmap-lru-1.2.jar.sha1
      - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/concurrentlinkedhashmap-lru-1.2.jar.sha1
    lucene/dev/branches/branch_4x/solr/licenses/concurrentlinkedhashmap-lru-LICENSE-ASL.txt
      - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/concurrentlinkedhashmap-lru-LICENSE-ASL.txt
    lucene/dev/branches/branch_4x/solr/licenses/concurrentlinkedhashmap-lru-NOTICE.txt
      - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/concurrentlinkedhashmap-lru-NOTICE.txt
    lucene/dev/branches/branch_4x/solr/licenses/hadoop-annotations-2.0.5-alpha.jar.sha1
      - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/hadoop-annotations-2.0.5-alpha.jar.sha1
    lucene/dev/branches/branch_4x/solr/licenses/hadoop-annotations-LICENSE-ASL.txt
      - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/hadoop-annotations-LICENSE-ASL.txt
    lucene/dev/branches/branch_4x/solr/licenses/hadoop-annotations-NOTICE.txt
      - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/hadoop-annotations-NOTICE.txt
    lucene/dev/branches/branch_4x/solr/licenses/hadoop-auth-2.0.5-alpha.jar.sha1
      - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/hadoop-auth-2.0.5-alpha.jar.sha1
    lucene/dev/branches/branch_4x/solr/licenses/hadoop-auth-LICENSE-ASL.txt
      - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/hadoop-auth-LICENSE-ASL.txt
    lucene/dev/branches/branch_4x/solr/licenses/hadoop-auth-NOTICE.txt
      - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/hadoop-auth-NOTICE.txt
    lucene/dev/branches/branch_4x/solr/licenses/hadoop-common-2.0.0-cdh4.2.0-SNAPSHOT-tests-NOTICE.txt
      - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/hadoop-common-2.0.0-cdh4.2.0-SNAPSHOT-tests-NOTICE.txt
    lucene/dev/branches/branch_4x/solr/licenses/hadoop-common-2.0.5-alpha-tests.jar.sha1
      - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/hadoop-common-2.0.5-alpha-tests.jar.sha1
    lucene/dev/branches/branch_4x/solr/licenses/hadoop-common-2.0.5-alpha.jar.sha1
      - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/hadoop-common-2.0.5-alpha.jar.sha1
    lucene/dev/branches/branch_4x/solr/licenses/hadoop-common-LICENSE-ASL.txt
      - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/hadoop-common-LICENSE-ASL.txt
    lucene/dev/branches/branch_4x/solr/licenses/hadoop-common-NOTICE.txt
      - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/hadoop-common-NOTICE.txt
    lucene/dev/branches/branch_4x/solr/licenses/hadoop-common-tests-LICENSE-ASL.txt
      - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/hadoop-common-tests-LICENSE-ASL.txt
    lucene/dev/branches/branch_4x/solr/licenses/hadoop-hdfs-2.0.5-alpha-tests.jar.sha1
      - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/hadoop-hdfs-2.0.5-alpha-tests.jar.sha1
    lucene/dev/branches/branch_4x/solr/licenses/hadoop-hdfs-2.0.5-alpha.jar.sha1
      - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/hadoop-hdfs-2.0.5-alpha.jar.sha1
    lucene/dev/branches/branch_4x/solr/licenses/hadoop-hdfs-LICENSE-ASL.txt
      - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/hadoop-hdfs-LICENSE-ASL.txt
    lucene/dev/branches/branch_4x/solr/licenses/hadoop-hdfs-NOTICE.txt
      - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/hadoop-hdfs-NOTICE.txt
    lucene/dev/branches/branch_4x/solr/licenses/hadoop-hdfs-tests-LICENSE-ASL.txt
      - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/hadoop-hdfs-tests-LICENSE-ASL.txt
    lucene/dev/branches/branch_4x/solr/licenses/hadoop-hdfs-tests-NOTICE.txt
      - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/hadoop-hdfs-tests-NOTICE.txt
    lucene/dev/branches/branch_4x/solr/licenses/jersey-core-1.16.jar.sha1
      - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/jersey-core-1.16.jar.sha1
    lucene/dev/branches/branch_4x/solr/licenses/jersey-core-LICENSE-CDDL.txt
      - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/jersey-core-LICENSE-CDDL.txt
    lucene/dev/branches/branch_4x/solr/licenses/jetty-6.1.26.jar.sha1
      - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/jetty-6.1.26.jar.sha1
    lucene/dev/branches/branch_4x/solr/licenses/jetty-util-6.1.26.jar.sha1
      - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/jetty-util-6.1.26.jar.sha1
    lucene/dev/branches/branch_4x/solr/licenses/log4j-1.2.17.jar.sha1
      - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/log4j-1.2.17.jar.sha1
    lucene/dev/branches/branch_4x/solr/licenses/protobuf-java-2.4.0a.jar.sha1
      - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/protobuf-java-2.4.0a.jar.sha1
    lucene/dev/branches/branch_4x/solr/licenses/protobuf-java-LICENSE-BSD.txt
      - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/protobuf-java-LICENSE-BSD.txt
    lucene/dev/branches/branch_4x/solr/licenses/protobuf-java-NOTICE.txt
      - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/protobuf-java-NOTICE.txt
    lucene/dev/branches/branch_4x/solr/licenses/zookeeper-3.4.5.jar.sha1
      - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/zookeeper-3.4.5.jar.sha1
Removed:
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/AssignShard.java
Modified:
    lucene/dev/branches/branch_4x/   (props changed)
    lucene/dev/branches/branch_4x/.gitignore
    lucene/dev/branches/branch_4x/build.xml
    lucene/dev/branches/branch_4x/lucene/   (props changed)
    lucene/dev/branches/branch_4x/lucene/common-build.xml   (contents, props changed)
    lucene/dev/branches/branch_4x/lucene/tools/   (props changed)
    lucene/dev/branches/branch_4x/lucene/tools/custom-tasks.xml
    lucene/dev/branches/branch_4x/lucene/tools/junit4/tests.policy
    lucene/dev/branches/branch_4x/solr/   (props changed)
    lucene/dev/branches/branch_4x/solr/CHANGES.txt
    lucene/dev/branches/branch_4x/solr/NOTICE.txt   (contents, props changed)
    lucene/dev/branches/branch_4x/solr/build.xml   (contents, props changed)
    lucene/dev/branches/branch_4x/solr/common-build.xml   (contents, props changed)
    lucene/dev/branches/branch_4x/solr/core/   (props changed)
    lucene/dev/branches/branch_4x/solr/core/ivy.xml
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/SolrLogFormatter.java
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/CloudDescriptor.java
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/Overseer.java
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/ZkController.java
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/CachingDirectoryFactory.java
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/ConfigSolr.java
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/ConfigSolrXml.java
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/ConfigSolrXmlOld.java
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/CoreContainer.java
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/EphemeralDirectoryFactory.java
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/SolrCore.java
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/ZkContainer.java
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/ReusedBufferedIndexOutput.java
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/hdfs/HdfsDirectory.java
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/hdfs/NullIndexOutput.java
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/TransactionLog.java
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/UpdateHandler.java
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/UpdateLog.java
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
    lucene/dev/branches/branch_4x/solr/core/src/test-files/log4j.properties
    lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml
    lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/solr-no-core.xml
    lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/solr.xml
    lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java
    lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java
    lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ClusterStateUpdateTest.java
    lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java
    lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java
    lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java
    lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java
    lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java
    lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java
    lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/search/TestRecovery.java
    lucene/dev/branches/branch_4x/solr/example/   (props changed)
    lucene/dev/branches/branch_4x/solr/example/resources/log4j.properties
    lucene/dev/branches/branch_4x/solr/example/solr/solr.xml
    lucene/dev/branches/branch_4x/solr/licenses/   (props changed)
    lucene/dev/branches/branch_4x/solr/solrj/   (props changed)
    lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java
    lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
    lucene/dev/branches/branch_4x/solr/solrj/src/test-files/solrj/log4j.properties
    lucene/dev/branches/branch_4x/solr/solrj/src/test-files/solrj/solr/shared/solr.xml
    lucene/dev/branches/branch_4x/solr/solrj/src/test-files/solrj/solr/solr.xml
    lucene/dev/branches/branch_4x/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/TestSolrProperties.java
    lucene/dev/branches/branch_4x/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestCoreAdmin.java
    lucene/dev/branches/branch_4x/solr/test-framework/   (props changed)
    lucene/dev/branches/branch_4x/solr/test-framework/ivy.xml
    lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
    lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDistribZkTestBase.java
    lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
    lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/util/TestHarness.java

Modified: lucene/dev/branches/branch_4x/.gitignore
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/.gitignore?rev=1498702&r1=1498701&r2=1498702&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/.gitignore (original)
+++ lucene/dev/branches/branch_4x/.gitignore Mon Jul  1 21:31:19 2013
@@ -1,4 +1,5 @@
-
+# hdfs
+/solr/example/hdfs
 *.jar
 
 # .

Modified: lucene/dev/branches/branch_4x/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/build.xml?rev=1498702&r1=1498701&r2=1498702&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/build.xml (original)
+++ lucene/dev/branches/branch_4x/build.xml Mon Jul  1 21:31:19 2013
@@ -205,7 +205,7 @@
     <!-- TODO: find a better way to exclude duplicate JAR files & fix the servlet-api mess! -->
     <pathconvert property="eclipse.fileset.libs" pathsep="|" dirsep="/">
       <fileset dir="${basedir}/lucene" includes="**/lib/*.jar" excludes="**/*servlet-api*.jar, analysis/uima/**, tools/**, build/**"/>
-      <fileset dir="${basedir}/solr" includes="**/lib/*.jar" excludes="core/lib/*servlet-api*.jar, contrib/analysis-extras/**, test-framework/**, build/**, dist/**, package/**" />
+      <fileset dir="${basedir}/solr" includes="**/lib/*.jar" excludes="core/lib/*servlet-api*.jar, contrib/analysis-extras/**, test-framework/lib/junit*, test-framework/lib/ant*, test-framework/lib/randomizedtesting*, build/**, dist/**, package/**" />
       <map from="${basedir}/" to=""/>
     </pathconvert>
     <xslt in="${ant.file}" out=".classpath" style="dev-tools/eclipse/dot.classpath.xsl" force="true">

Modified: lucene/dev/branches/branch_4x/lucene/common-build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/common-build.xml?rev=1498702&r1=1498701&r2=1498702&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/common-build.xml (original)
+++ lucene/dev/branches/branch_4x/lucene/common-build.xml Mon Jul  1 21:31:19 2013
@@ -359,7 +359,7 @@
   <target name="resolve" depends="ivy-availability-check,ivy-configure">
     <!-- todo, make this a property or something. 
          only special cases need bundles -->
-    <ivy:retrieve type="jar,bundle" log="download-only" 
+    <ivy:retrieve type="jar,bundle,tests" log="download-only" 
                   conf="${ivy.default.configuration}" sync="${ivy.sync}"/>
   </target>
 

Modified: lucene/dev/branches/branch_4x/lucene/tools/custom-tasks.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/tools/custom-tasks.xml?rev=1498702&r1=1498701&r2=1498702&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/tools/custom-tasks.xml (original)
+++ lucene/dev/branches/branch_4x/lucene/tools/custom-tasks.xml Mon Jul  1 21:31:19 2013
@@ -43,7 +43,7 @@
 
     <!-- Typical version patterns. -->
     <replaceregex pattern="\.rc[0-9]+" replace="" flags="gi" />
-    <replaceregex pattern="\-(r)?([0-9\-\_\.])+(b(eta)?([0-9\-\.])*)?$" replace="" flags="gi" />
+    <replaceregex pattern="\-(r)?([0-9\-\_\.])+((b(eta)?)|((a(lpha)?))([0-9\-\.])*)?(\-tests)?$" replace="" flags="gi" />
 
     <!-- git hashcode pattern: its always 40 chars right? -->
     <replaceregex pattern="\-[a-z0-9]{40,40}$" replace="" flags="gi" />

Modified: lucene/dev/branches/branch_4x/lucene/tools/junit4/tests.policy
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/tools/junit4/tests.policy?rev=1498702&r1=1498701&r2=1498702&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/tools/junit4/tests.policy (original)
+++ lucene/dev/branches/branch_4x/lucene/tools/junit4/tests.policy Mon Jul  1 21:31:19 2013
@@ -59,6 +59,10 @@ grant {
   permission javax.management.MBeanPermission "*", "*";
   permission javax.management.MBeanServerPermission "*";
   permission javax.management.MBeanTrustPermission "*";
+  permission javax.security.auth.AuthPermission "*";
+  permission javax.security.auth.PrivateCredentialPermission "org.apache.hadoop.security.Credentials * \"*\"", "read";
+  permission java.security.SecurityPermission "putProviderProperty.SaslPlainServer";
+  permission java.security.SecurityPermission "insertProvider.SaslPlainServer";
   
   // TIKA uses BouncyCastle and that registers new provider for PDF parsing + MSOffice parsing. Maybe report as bug!
   permission java.security.SecurityPermission "putProviderProperty.BC";

Modified: lucene/dev/branches/branch_4x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/CHANGES.txt?rev=1498702&r1=1498701&r2=1498702&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/solr/CHANGES.txt Mon Jul  1 21:31:19 2013
@@ -113,6 +113,9 @@ New Features
 * SOLR-4897: Add solr/example/example-schemaless/, an example config set
   for schemaless mode. (Steve Rowe)
 
+* SOLR-4916: Add support to write and read Solr index files and transaction log
+  files to and from HDFS. (phunt, Mark Miller, Greg Chanan)
+
 Bug Fixes
 ----------------------
 

Modified: lucene/dev/branches/branch_4x/solr/NOTICE.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/NOTICE.txt?rev=1498702&r1=1498701&r2=1498702&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/NOTICE.txt (original)
+++ lucene/dev/branches/branch_4x/solr/NOTICE.txt Mon Jul  1 21:31:19 2013
@@ -88,6 +88,7 @@ including, but not limited to:
  - Apache Jakarta Regexp
  - Apache Commons
  - Apache Xerces
+ - Apache Blur
 
 ICU4J, (under analysis/icu) is licensed under an MIT styles license
 and Copyright (c) 1995-2008 International Business Machines Corporation and others
@@ -108,6 +109,9 @@ http://bitbucket.org/jpbarrette/moman/ov
 The class org.apache.lucene.util.WeakIdentityMap was derived from
 the Apache CXF project and is Apache License 2.0.
 
+The HdfsDirectory and BlockDirectory were derived from
+the Apache Blur incubating project and are Apache License 2.0.
+
 The Google Code Prettify is Apache License 2.0.
 See http://code.google.com/p/google-code-prettify/
 

Modified: lucene/dev/branches/branch_4x/solr/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/build.xml?rev=1498702&r1=1498701&r2=1498702&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/build.xml (original)
+++ lucene/dev/branches/branch_4x/solr/build.xml Mon Jul  1 21:31:19 2013
@@ -40,7 +40,7 @@
   <!-- ========================================================================= -->
  
   <target name="example" description="Creates a runnable example configuration."
-          depends="dist-contrib,dist-war">
+          depends="dist-contrib,dist-war,setup-alt-examples">
     <copy file="${dist}/${fullnamever}.war"
           tofile="${example}/webapps/${ant.project.name}.war"/>
     <jar destfile="${example}/exampledocs/post.jar"
@@ -56,6 +56,15 @@
     <echo>See ${example}/README.txt for how to run the Solr example configuration.</echo>
   </target>
 
+  <target name="setup-alt-examples">
+    <copy todir="${example}/hdfs" overwrite="true">
+      <fileset dir="${example}/solr"/>
+    </copy>
+    <copy todir="${example}/hdfs/collection1/conf" overwrite="true">
+      <fileset dir="${example}/alt-configs/hdfs"/>
+    </copy>
+  </target>
+  
   <target name="run-example" depends="example"
           description="Run Solr interactively, via Jetty.  -Dexample.debug=true to enable JVM debugger">
     <property name="example.solr.home" location="example/solr"/>

Modified: lucene/dev/branches/branch_4x/solr/common-build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/common-build.xml?rev=1498702&r1=1498701&r2=1498702&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/common-build.xml (original)
+++ lucene/dev/branches/branch_4x/solr/common-build.xml Mon Jul  1 21:31:19 2013
@@ -96,6 +96,12 @@
 
   <path id="solr.test.base.classpath">
     <pathelement path="${common-solr.dir}/build/solr-test-framework/classes/java"/>
+    <fileset dir="${common-solr.dir}/test-framework/lib">
+      <include name="*.jar"/>
+      <exclude name="junit-*.jar" />
+      <exclude name="randomizedtesting-runner-*.jar" />
+      <exclude name="ant*.jar" />
+    </fileset>
   	<pathelement path="${build.dir}/test-files"/>
   	<path refid="test.base.classpath"/>
   </path>

Modified: lucene/dev/branches/branch_4x/solr/core/ivy.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/ivy.xml?rev=1498702&r1=1498701&r2=1498702&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/ivy.xml (original)
+++ lucene/dev/branches/branch_4x/solr/core/ivy.xml Mon Jul  1 21:31:19 2013
@@ -16,6 +16,9 @@
    specific language governing permissions and limitations
    under the License.    
 -->
+<!DOCTYPE ivy-module [
+  <!ENTITY hadoop.version "2.0.5-alpha">
+]>
 <ivy-module version="2.0">
     <info organisation="org.apache.solr" module="core"/>
 
@@ -33,6 +36,14 @@
       <dependency org="org.restlet.jee" name="org.restlet" rev="2.1.1" transitive="false"/>
       <dependency org="org.restlet.jee" name="org.restlet.ext.servlet" rev="2.1.1" transitive="false"/>
       <dependency org="joda-time" name="joda-time" rev="2.2" transitive="false"/>
-      <exclude org="*" ext="*" matcher="regexp" type="${ivy.exclude.types}"/>
+      
+      <dependency org="org.apache.hadoop" name="hadoop-common" rev="&hadoop.version;" transitive="false"/>
+      <dependency org="org.apache.hadoop" name="hadoop-hdfs" rev="&hadoop.version;" transitive="false"/>
+      <dependency org="org.apache.hadoop" name="hadoop-annotations" rev="&hadoop.version;" transitive="false"/>
+      <dependency org="org.apache.hadoop" name="hadoop-auth" rev="&hadoop.version;" transitive="false"/>
+      <dependency org="commons-configuration" name="commons-configuration" rev="1.6" transitive="false"/>
+      <dependency org="com.google.protobuf" name="protobuf-java" rev="2.4.0a" transitive="false"/>
+      <dependency org="com.googlecode.concurrentlinkedhashmap" name="concurrentlinkedhashmap-lru" rev="1.2" transitive="false"/>
+      <exclude org="*" ext="*" matcher="regexp" type="${ivy.exclude.types}"/> 
     </dependencies>
 </ivy-module>

Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/SolrLogFormatter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/SolrLogFormatter.java?rev=1498702&r1=1498701&r2=1498702&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/SolrLogFormatter.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/SolrLogFormatter.java Mon Jul  1 21:31:19 2013
@@ -17,21 +17,25 @@ package org.apache.solr;
  * limitations under the License.
  */
 
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.WeakHashMap;
+import java.util.logging.ConsoleHandler;
+import java.util.logging.Formatter;
+import java.util.logging.Handler;
+import java.util.logging.Level;
+import java.util.logging.LogRecord;
+import java.util.logging.Logger;
+
 import org.apache.solr.cloud.ZkController;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.cloud.Replica;
-import org.apache.solr.common.cloud.ZkStateReader;
 import org.apache.solr.core.SolrCore;
 import org.apache.solr.request.SolrQueryRequest;
 import org.apache.solr.request.SolrRequestInfo;
 import org.slf4j.LoggerFactory;
 
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.WeakHashMap;
-import java.util.logging.*;
-
 public class SolrLogFormatter extends Formatter {
 
   /** Add this interface to a thread group and the string returned by
@@ -259,7 +263,7 @@ sb.append("(group_name=").append(tg.getN
 
   private Map<String,Object> getReplicaProps(ZkController zkController, SolrCore core) {
     final String collection = core.getCoreDescriptor().getCloudDescriptor().getCollectionName();
-    Replica replica = zkController.getClusterState().getReplica(collection, zkController.getCoreNodeName(core.getCoreDescriptor()));
+    Replica replica = zkController.getClusterState().getReplica(collection, core.getCoreDescriptor().getCloudDescriptor().getCoreNodeName());
     if(replica!=null) {
       return replica.getProperties();
     }

Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java?rev=1498702&r1=1498701&r2=1498702&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java Mon Jul  1 21:31:19 2013
@@ -85,6 +85,7 @@ public class JettySolrRunner {
   private String shards;
 
   private String dataDir;
+  private String solrUlogDir;
   
   private volatile boolean startedBefore = false;
 
@@ -359,6 +360,9 @@ public class JettySolrRunner {
     if( dataDir != null) {
       System.setProperty("solr.data.dir", dataDir);
     }
+    if( solrUlogDir != null) {
+      System.setProperty("solr.ulog.dir", solrUlogDir);
+    }
     if(shards != null) {
       System.setProperty("shard", shards);
     }
@@ -382,6 +386,8 @@ public class JettySolrRunner {
     System.clearProperty("shard");
     System.clearProperty("solr.data.dir");
     System.clearProperty("coreNodeName");
+    System.clearProperty("solr.ulog.dir");
+
   }
 
   public void stop() throws Exception {
@@ -485,6 +491,10 @@ public class JettySolrRunner {
   public void setDataDir(String dataDir) {
     this.dataDir = dataDir;
   }
+  
+  public void setUlogDir(String ulogDir) {
+    this.solrUlogDir = ulogDir;
+  }
 
   public void setCoreNodeName(String coreNodeName) {
     this.coreNodeName = coreNodeName;

Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/CloudDescriptor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/CloudDescriptor.java?rev=1498702&r1=1498701&r2=1498702&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/CloudDescriptor.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/CloudDescriptor.java Mon Jul  1 21:31:19 2013
@@ -44,6 +44,10 @@ public class CloudDescriptor {
   public boolean isLeader() {
     return isLeader;
   }
+  
+  public void setLeader(boolean isLeader) {
+    this.isLeader = isLeader;
+  }
 
   public void setShardId(String shardId) {
     this.shardId = shardId;

Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java?rev=1498702&r1=1498701&r2=1498702&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java Mon Jul  1 21:31:19 2013
@@ -47,9 +47,9 @@ public abstract class ElectionContext {
   String leaderSeqPath;
   private SolrZkClient zkClient;
   
-  public ElectionContext(final String shardZkNodeName,
+  public ElectionContext(final String coreNodeName,
       final String electionPath, final String leaderPath, final ZkNodeProps leaderProps, final SolrZkClient zkClient) {
-    this.id = shardZkNodeName;
+    this.id = coreNodeName;
     this.electionPath = electionPath;
     this.leaderPath = leaderPath;
     this.leaderProps = leaderProps;
@@ -78,8 +78,8 @@ class ShardLeaderElectionContextBase ext
   protected LeaderElector leaderElector;
 
   public ShardLeaderElectionContextBase(LeaderElector leaderElector, final String shardId,
-      final String collection, final String shardZkNodeName, ZkNodeProps props, ZkStateReader zkStateReader) {
-    super(shardZkNodeName, ZkStateReader.COLLECTIONS_ZKNODE + "/" + collection + "/leader_elect/"
+      final String collection, final String coreNodeName, ZkNodeProps props, ZkStateReader zkStateReader) {
+    super(coreNodeName, ZkStateReader.COLLECTIONS_ZKNODE + "/" + collection + "/leader_elect/"
         + shardId, ZkStateReader.getShardLeadersPath(collection, shardId),
         props, zkStateReader.getZkClient());
     this.leaderElector = leaderElector;
@@ -95,7 +95,7 @@ class ShardLeaderElectionContextBase ext
     zkClient.makePath(leaderPath, ZkStateReader.toJSON(leaderProps),
         CreateMode.EPHEMERAL, true);
     assert shardId != null;
-    ZkNodeProps m = ZkNodeProps.fromKeyVals(Overseer.QUEUE_OPERATION, "leader",
+    ZkNodeProps m = ZkNodeProps.fromKeyVals(Overseer.QUEUE_OPERATION, ZkStateReader.LEADER_PROP,
         ZkStateReader.SHARD_ID_PROP, shardId, ZkStateReader.COLLECTION_PROP,
         collection, ZkStateReader.BASE_URL_PROP, leaderProps.getProperties()
             .get(ZkStateReader.BASE_URL_PROP), ZkStateReader.CORE_NAME_PROP,
@@ -119,8 +119,8 @@ final class ShardLeaderElectionContext e
   
   public ShardLeaderElectionContext(LeaderElector leaderElector, 
       final String shardId, final String collection,
-      final String shardZkNodeName, ZkNodeProps props, ZkController zkController, CoreContainer cc) {
-    super(leaderElector, shardId, collection, shardZkNodeName, props,
+      final String coreNodeName, ZkNodeProps props, ZkController zkController, CoreContainer cc) {
+    super(leaderElector, shardId, collection, coreNodeName, props,
         zkController.getZkStateReader());
     this.zkController = zkController;
     this.cc = cc;
@@ -138,12 +138,12 @@ final class ShardLeaderElectionContext e
   @Override
   void runLeaderProcess(boolean weAreReplacement) throws KeeperException,
       InterruptedException, IOException {
-    log.info("Running the leader process.");
+    log.info("Running the leader process for shard " + shardId);
     
     String coreName = leaderProps.getStr(ZkStateReader.CORE_NAME_PROP);
     
     // clear the leader in clusterstate
-    ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION, "leader",
+    ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION, ZkStateReader.LEADER_PROP,
         ZkStateReader.SHARD_ID_PROP, shardId, ZkStateReader.COLLECTION_PROP,
         collection);
     Overseer.getInQueue(zkClient).offer(ZkStateReader.toJSON(m));
@@ -243,8 +243,8 @@ final class ShardLeaderElectionContext e
       }
 
       log.info("I am the new leader: "
-          + ZkCoreNodeProps.getCoreUrl(leaderProps));
-      core.getCoreDescriptor().getCloudDescriptor().isLeader = true;
+          + ZkCoreNodeProps.getCoreUrl(leaderProps) + " " + shardId);
+      core.getCoreDescriptor().getCloudDescriptor().setLeader(true);
     } finally {
       if (core != null) {
         core.close();
@@ -254,16 +254,17 @@ final class ShardLeaderElectionContext e
     try {
       super.runLeaderProcess(weAreReplacement);
     } catch (Throwable t) {
+      SolrException.log(log, "There was a problem trying to register as the leader", t);
+      cancelElection();
       try {
         core = cc.getCore(coreName);
         if (core == null) {
-          cancelElection();
           throw new SolrException(ErrorCode.SERVER_ERROR,
               "Fatal Error, SolrCore not found:" + coreName + " in "
                   + cc.getCoreNames());
         }
         
-        core.getCoreDescriptor().getCloudDescriptor().isLeader = false;
+        core.getCoreDescriptor().getCloudDescriptor().setLeader(false);
         
         // we could not publish ourselves as leader - rejoin election
         rejoinLeaderElection(leaderSeqPath, core);
@@ -332,7 +333,7 @@ final class ShardLeaderElectionContext e
           return;
         } else {
           if (cnt % 40 == 0) {
-            log.info("Waiting until we see more replicas up: total="
+            log.info("Waiting until we see more replicas up for shard " + shardId + ": total="
               + slices.getReplicasMap().size() + " found=" + found
               + " timeoutin=" + (timeoutAt - System.currentTimeMillis()));
           }

Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/Overseer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/Overseer.java?rev=1498702&r1=1498701&r2=1498702&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/Overseer.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/Overseer.java Mon Jul  1 21:31:19 2013
@@ -50,15 +50,16 @@ import org.slf4j.LoggerFactory;
  */
 public class Overseer {
   public static final String QUEUE_OPERATION = "operation";
+  public static final String DELETECORE = "deletecore";
   public static final String REMOVECOLLECTION = "removecollection";
   
   private static final int STATE_UPDATE_DELAY = 1500;  // delay between cloud state updates
 
+
   private static Logger log = LoggerFactory.getLogger(Overseer.class);
   
   private class ClusterStateUpdater implements Runnable, ClosableThread {
     
-    private static final String DELETECORE = "deletecore";
     private final ZkStateReader reader;
     private final SolrZkClient zkClient;
     private final String myId;
@@ -267,8 +268,14 @@ public class Overseer {
         final String collection = message.getStr(ZkStateReader.COLLECTION_PROP);
         String coreNodeName = message.getStr(ZkStateReader.CORE_NODE_NAME_PROP);
         if (coreNodeName == null) {
-          // it must be the default then
-          coreNodeName = message.getStr(ZkStateReader.NODE_NAME_PROP) + "_" + message.getStr(ZkStateReader.CORE_NAME_PROP);
+          coreNodeName = getAssignedCoreNodeName(state, message);
+          if (coreNodeName != null) {
+            log.info("node=" + coreNodeName + " is already registered");
+          } else {
+            // if coreNodeName is null, auto assign one
+            coreNodeName = Assign.assignNode(collection, state);
+          }
+          message.getProperties().put(ZkStateReader.CORE_NODE_NAME_PROP, coreNodeName);
         }
         Integer numShards = message.getStr(ZkStateReader.NUM_SHARDS_PROP)!=null?Integer.parseInt(message.getStr(ZkStateReader.NUM_SHARDS_PROP)):null;
         log.info("Update state numShards={} message={}", numShards, message);
@@ -281,7 +288,6 @@ public class Overseer {
         // use the provided non null shardId
         String sliceName = message.getStr(ZkStateReader.SHARD_ID_PROP);
         if (sliceName == null) {
-          //String nodeName = message.getStr(ZkStateReader.NODE_NAME_PROP);
           //get shardId from ClusterState
           sliceName = getAssignedId(state, coreNodeName, message);
           if (sliceName != null) {
@@ -295,8 +301,8 @@ public class Overseer {
             numShards = state.getCollectionStates().get(collection).getSlices().size();
             log.info("Collection already exists with " + ZkStateReader.NUM_SHARDS_PROP + "=" + numShards);
           }
-          sliceName = AssignShard.assignShard(collection, state, numShards);
-          log.info("Assigning new node to shard=" + sliceName);
+          sliceName = Assign.assignShard(collection, state, numShards);
+          log.info("Assigning new node to shard shard=" + sliceName);
         }
 
         Slice slice = state.getSlice(collection, sliceName);
@@ -320,8 +326,11 @@ public class Overseer {
           }
         }
 
-        // we don't put num_shards in the clusterstate
+        // we don't put these in the clusterstate
           replicaProps.remove(ZkStateReader.NUM_SHARDS_PROP);
+          replicaProps.remove(ZkStateReader.CORE_NODE_NAME_PROP);
+          replicaProps.remove(ZkStateReader.SHARD_ID_PROP);
+          replicaProps.remove(ZkStateReader.COLLECTION_PROP);
           replicaProps.remove(QUEUE_OPERATION);
           
           // remove any props with null values
@@ -417,6 +426,26 @@ public class Overseer {
         return null;
       }
       
+      private String getAssignedCoreNodeName(ClusterState state, ZkNodeProps message) {
+        Collection<Slice> slices = state.getSlices(message.getStr(ZkStateReader.COLLECTION_PROP));
+        if (slices != null) {
+          for (Slice slice : slices) {
+            for (Replica replica : slice.getReplicas()) {
+              String baseUrl = replica.getStr(ZkStateReader.BASE_URL_PROP);
+              String core = replica.getStr(ZkStateReader.CORE_NAME_PROP);
+              
+              String msgBaseUrl = message.getStr(ZkStateReader.BASE_URL_PROP);
+              String msgCore = message.getStr(ZkStateReader.CORE_NAME_PROP);
+              
+              if (baseUrl.equals(msgBaseUrl) && core.equals(msgCore)) {
+                return replica.getName();
+              }
+            }
+          }
+        }
+        return null;
+      }
+      
       private ClusterState updateSlice(ClusterState state, String collectionName, Slice slice) {
         // System.out.println("###!!!### OLD CLUSTERSTATE: " + JSONUtil.toJSON(state.getCollectionStates()));
         // System.out.println("Updating slice:" + slice);
@@ -526,10 +555,6 @@ public class Overseer {
       private ClusterState removeCore(final ClusterState clusterState, ZkNodeProps message) {
         
         String cnn = message.getStr(ZkStateReader.CORE_NODE_NAME_PROP);
-        if (cnn == null) {
-          // it must be the default then
-          cnn = message.getStr(ZkStateReader.NODE_NAME_PROP) + "_" + message.getStr(ZkStateReader.CORE_NAME_PROP);
-        }
 
         final String collection = message.getStr(ZkStateReader.COLLECTION_PROP);
 

Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java?rev=1498702&r1=1498701&r2=1498702&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java Mon Jul  1 21:31:19 2013
@@ -400,10 +400,11 @@ public class OverseerCollectionProcessor
 
         // wait for parent leader to acknowledge the sub-shard core
         log.info("Asking parent leader to wait for: " + subShardName + " to be alive on: " + nodeName);
+        String coreNodeName = waitForCoreNodeName(collection, zkStateReader.getZkClient().getBaseUrlForNodeName(nodeName), subShardName);
         CoreAdminRequest.WaitForState cmd = new CoreAdminRequest.WaitForState();
         cmd.setCoreName(subShardName);
         cmd.setNodeName(nodeName);
-        cmd.setCoreNodeName(nodeName + "_" + subShardName);
+        cmd.setCoreNodeName(coreNodeName);
         cmd.setState(ZkStateReader.ACTIVE);
         cmd.setCheckLive(true);
         cmd.setOnlyIfLeader(true);
@@ -506,12 +507,13 @@ public class OverseerCollectionProcessor
 
           sendShardRequest(subShardNodeName, params);
 
+          String coreNodeName = waitForCoreNodeName(collection, zkStateReader.getZkClient().getBaseUrlForNodeName(subShardNodeName), shardName);
           // wait for the replicas to be seen as active on sub shard leader
           log.info("Asking sub shard leader to wait for: " + shardName + " to be alive on: " + subShardNodeName);
           CoreAdminRequest.WaitForState cmd = new CoreAdminRequest.WaitForState();
           cmd.setCoreName(subShardNames.get(i-1));
           cmd.setNodeName(subShardNodeName);
-          cmd.setCoreNodeName(subShardNodeName + "_" + shardName);
+          cmd.setCoreNodeName(coreNodeName);
           cmd.setState(ZkStateReader.ACTIVE);
           cmd.setCheckLive(true);
           cmd.setOnlyIfLeader(true);
@@ -545,6 +547,35 @@ public class OverseerCollectionProcessor
       throw new SolrException(ErrorCode.SERVER_ERROR, null, e);
     }
   }
+  
+  private String waitForCoreNodeName(DocCollection collection, String msgBaseUrl, String msgCore) {
+    int retryCount = 320;
+    while (retryCount-- > 0) {
+      Map<String,Slice> slicesMap = zkStateReader.getClusterState()
+          .getSlicesMap(collection.getName());
+      if (slicesMap != null) {
+        
+        for (Slice slice : slicesMap.values()) {
+          for (Replica replica : slice.getReplicas()) {
+            // TODO: for really large clusters, we could 'index' on this
+            
+            String baseUrl = replica.getStr(ZkStateReader.BASE_URL_PROP);
+            String core = replica.getStr(ZkStateReader.CORE_NAME_PROP);
+            
+            if (baseUrl.equals(msgBaseUrl) && core.equals(msgCore)) {
+              return replica.getName();
+            }
+          }
+        }
+      }
+      try {
+        Thread.sleep(1000);
+      } catch (InterruptedException e) {
+        Thread.currentThread().interrupt();
+      }
+    }
+    throw new SolrException(ErrorCode.SERVER_ERROR, "Could not find coreNodeName");
+  }
 
   private void collectShardResponses(NamedList results, boolean abortOnError, String msgOnError) {
     ShardResponse srsp;

Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java?rev=1498702&r1=1498701&r2=1498702&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java Mon Jul  1 21:31:19 2013
@@ -91,7 +91,7 @@ public class RecoveryStrategy extends Th
     zkController = cc.getZkController();
     zkStateReader = zkController.getZkStateReader();
     baseUrl = zkController.getBaseUrl();
-    coreZkNodeName = zkController.getCoreNodeName(cd);
+    coreZkNodeName = cd.getCloudDescriptor().getCoreNodeName();
   }
 
   public void setRecoveringAfterStartup(boolean recoveringAfterStartup) {
@@ -325,10 +325,10 @@ public class RecoveryStrategy extends Th
         String ourUrl = ZkCoreNodeProps.getCoreUrl(baseUrl, coreName);
 
         boolean isLeader = leaderUrl.equals(ourUrl);
-        if (isLeader && !cloudDesc.isLeader) {
+        if (isLeader && !cloudDesc.isLeader()) {
           throw new SolrException(ErrorCode.SERVER_ERROR, "Cloud state still says we are leader.");
         }
-        if (cloudDesc.isLeader) {
+        if (cloudDesc.isLeader()) {
           // we are now the leader - no one else must have been suitable
           log.warn("We have not yet recovered - but we are now the leader! core=" + coreName);
           log.info("Finished recovery process. core=" + coreName);

Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java?rev=1498702&r1=1498701&r2=1498702&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java Mon Jul  1 21:31:19 2013
@@ -146,8 +146,7 @@ public class SyncStrategy {
   private boolean syncWithReplicas(ZkController zkController, SolrCore core,
       ZkNodeProps props, String collection, String shardId) {
     List<ZkCoreNodeProps> nodes = zkController.getZkStateReader()
-        .getReplicaProps(collection, shardId,
-            zkController.getCoreNodeName(core.getCoreDescriptor()),
+        .getReplicaProps(collection, shardId,core.getCoreDescriptor().getCloudDescriptor().getCoreNodeName(),
             props.getStr(ZkStateReader.CORE_NAME_PROP));
     
     if (nodes == null) {
@@ -175,7 +174,7 @@ public class SyncStrategy {
     List<ZkCoreNodeProps> nodes = zkController
         .getZkStateReader()
         .getReplicaProps(collection, shardId,
-            zkController.getCoreNodeName(cd),
+            cd.getCloudDescriptor().getCoreNodeName(),
             leaderProps.getStr(ZkStateReader.CORE_NAME_PROP));
     if (nodes == null) {
       log.info(ZkCoreNodeProps.getCoreUrl(leaderProps) + " has no replicas");

Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/ZkController.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/ZkController.java?rev=1498702&r1=1498701&r2=1498702&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/ZkController.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/ZkController.java Mon Jul  1 21:31:19 2013
@@ -132,6 +132,8 @@ public final class ZkController {
   protected volatile Overseer overseer;
 
   private String leaderVoteWait;
+  
+  private boolean genericCoreNodeNames;
 
   private int clientTimeout;
 
@@ -140,11 +142,11 @@ public final class ZkController {
   private UpdateShardHandler updateShardHandler;
 
   public ZkController(final CoreContainer cc, String zkServerAddress, int zkClientTimeout, int zkClientConnectTimeout, String localHost, String locaHostPort,
-      String localHostContext, String leaderVoteWait, int distribUpdateConnTimeout, int distribUpdateSoTimeout, final CurrentCoreDescriptorProvider registerOnReconnect) throws InterruptedException,
+      String localHostContext, String leaderVoteWait, boolean genericCoreNodeNames, int distribUpdateConnTimeout, int distribUpdateSoTimeout, final CurrentCoreDescriptorProvider registerOnReconnect) throws InterruptedException,
       TimeoutException, IOException {
     if (cc == null) throw new IllegalArgumentException("CoreContainer cannot be null.");
     this.cc = cc;
-
+    this.genericCoreNodeNames = genericCoreNodeNames;
     // be forgiving and strip this off leading/trailing slashes
     // this allows us to support users specifying hostContext="/" in 
     // solr.xml to indicate the root context, instead of hostContext="" 
@@ -254,9 +256,9 @@ public final class ZkController {
       // before registering as live, make sure everyone is in a
       // down state
       for (CoreDescriptor descriptor : descriptors) {
-        final String coreZkNodeName = getCoreNodeName(descriptor);
+        final String coreZkNodeName = descriptor.getCloudDescriptor().getCoreNodeName();
         try {
-          descriptor.getCloudDescriptor().isLeader = false;
+          descriptor.getCloudDescriptor().setLeader(false);
           publish(descriptor, ZkStateReader.DOWN, updateLastPublished);
         } catch (Exception e) {
           if (isClosed) {
@@ -323,7 +325,7 @@ public final class ZkController {
         .getCurrentDescriptors();
     if (descriptors != null) {
       for (CoreDescriptor descriptor : descriptors) {
-        descriptor.getCloudDescriptor().isLeader = false;
+        descriptor.getCloudDescriptor().setLeader(false);
       }
     }
   }
@@ -544,7 +546,6 @@ public final class ZkController {
           if (replica.getNodeName().equals(getNodeName())
               && !(replica.getStr(ZkStateReader.STATE_PROP)
                   .equals(ZkStateReader.DOWN))) {
-            assert replica.getStr(ZkStateReader.SHARD_ID_PROP) != null;
             ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION, "state",
                 ZkStateReader.STATE_PROP, ZkStateReader.DOWN,
                 ZkStateReader.BASE_URL_PROP, getBaseUrl(),
@@ -555,8 +556,7 @@ public final class ZkController {
                 ZkStateReader.NODE_NAME_PROP, getNodeName(),
                 ZkStateReader.SHARD_ID_PROP,
                 replica.getStr(ZkStateReader.SHARD_ID_PROP),
-                ZkStateReader.COLLECTION_PROP,
-                replica.getStr(ZkStateReader.COLLECTION_PROP),
+                ZkStateReader.COLLECTION_PROP, collectionName,
                 ZkStateReader.CORE_NODE_NAME_PROP, replica.getName());
             updatedNodes.add(replica.getStr(ZkStateReader.CORE_NAME_PROP));
             overseerJobQueue.offer(ZkStateReader.toJSON(m));
@@ -735,7 +735,8 @@ public final class ZkController {
     final CloudDescriptor cloudDesc = desc.getCloudDescriptor();
     final String collection = cloudDesc.getCollectionName();
 
-    final String coreZkNodeName = getCoreNodeName(desc);
+    final String coreZkNodeName = desc.getCloudDescriptor().getCoreNodeName();
+    assert coreZkNodeName != null : "we should have a coreNodeName by now";
     
     String shardId = cloudDesc.getShardId();
 
@@ -923,16 +924,16 @@ public final class ZkController {
     props.put(ZkStateReader.CORE_NAME_PROP, cd.getName());
     props.put(ZkStateReader.NODE_NAME_PROP, getNodeName());
     
-    final String coreZkNodeName = getCoreNodeName(cd);
+    final String coreNodeName = cd.getCloudDescriptor().getCoreNodeName();
     ZkNodeProps ourProps = new ZkNodeProps(props);
     String collection = cd.getCloudDescriptor()
         .getCollectionName();
     
     ElectionContext context = new ShardLeaderElectionContext(leaderElector, shardId,
-        collection, coreZkNodeName, ourProps, this, cc);
+        collection, coreNodeName, ourProps, this, cc);
 
     leaderElector.setup(context);
-    electionContexts.put(coreZkNodeName, context);
+    electionContexts.put(coreNodeName, context);
     leaderElector.joinElection(context, false);
   }
 
@@ -1017,7 +1018,7 @@ public final class ZkController {
 
     final CloudDescriptor cloudDesc = desc.getCloudDescriptor();
     
-    final String shardId = state.getShardId(coreNodeName);
+    final String shardId = state.getShardId(getBaseUrl(), desc.getName());
 
     if (shardId != null) {
       cloudDesc.setShardId(shardId);
@@ -1028,16 +1029,21 @@ public final class ZkController {
 
   public void unregister(String coreName, CoreDescriptor cd)
       throws InterruptedException, KeeperException {
-    final String zkNodeName = getCoreNodeName(cd);
-    ElectionContext context = electionContexts.remove(zkNodeName);
+    final String coreNodeName = cd.getCloudDescriptor().getCoreNodeName();
+    ElectionContext context = electionContexts.remove(coreNodeName);
+    
+    assert context != null : coreNodeName;
+    
     if (context != null) {
       context.cancelElection();
     }
+    CloudDescriptor cloudDescriptor = cd.getCloudDescriptor();
     
     ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION,
-        "deletecore", ZkStateReader.CORE_NAME_PROP, coreName,
+        Overseer.DELETECORE, ZkStateReader.CORE_NAME_PROP, coreName,
         ZkStateReader.NODE_NAME_PROP, getNodeName(),
-        ZkStateReader.COLLECTION_PROP, cd.getCloudDescriptor().getCollectionName());
+        ZkStateReader.COLLECTION_PROP, cloudDescriptor.getCollectionName(),
+        ZkStateReader.CORE_NODE_NAME_PROP, coreNodeName);
     overseerJobQueue.offer(ZkStateReader.toJSON(m));
   }
   
@@ -1206,13 +1212,60 @@ public final class ZkController {
     return zkStateReader;
   }
 
-  private String doGetShardIdProcess(String coreName, CoreDescriptor descriptor) {
-    final String coreNodeName = getCoreNodeName(descriptor);
+  private void doGetShardIdAndNodeNameProcess(CoreDescriptor cd) {
+    final String coreNodeName = cd.getCloudDescriptor().getCoreNodeName();
+
+    if (coreNodeName != null) {
+      waitForShardId(cd);
+    } else {
+      // if no explicit coreNodeName, we want to match by base url and core name
+      waitForCoreNodeName(cd);
+      waitForShardId(cd);
+    }
+  }
+
+  private void waitForCoreNodeName(CoreDescriptor descriptor) {
     int retryCount = 320;
+    log.info("look for our core node name");
     while (retryCount-- > 0) {
-      final String shardId = zkStateReader.getClusterState().getShardId(coreNodeName);
+      Map<String,Slice> slicesMap = zkStateReader.getClusterState()
+          .getSlicesMap(descriptor.getCloudDescriptor().getCollectionName());
+      if (slicesMap != null) {
+        
+        for (Slice slice : slicesMap.values()) {
+          for (Replica replica : slice.getReplicas()) {
+            // TODO: for really large clusters, we could 'index' on this
+            
+            String baseUrl = replica.getStr(ZkStateReader.BASE_URL_PROP);
+            String core = replica.getStr(ZkStateReader.CORE_NAME_PROP);
+            
+            String msgBaseUrl = getBaseUrl();
+            String msgCore = descriptor.getName();
+
+            if (baseUrl.equals(msgBaseUrl) && core.equals(msgCore)) {
+              descriptor.getCloudDescriptor()
+                  .setCoreNodeName(replica.getName());
+              return;
+            }
+          }
+        }
+      }
+      try {
+        Thread.sleep(1000);
+      } catch (InterruptedException e) {
+        Thread.currentThread().interrupt();
+      }
+    }
+  }
+
+  private void waitForShardId(CoreDescriptor cd) {
+    log.info("waiting to find shard id in clusterstate for " + cd.getName());
+    int retryCount = 320;
+    while (retryCount-- > 0) {
+      final String shardId = zkStateReader.getClusterState().getShardId(getBaseUrl(), cd.getName());
       if (shardId != null) {
-        return shardId;
+        cd.getCloudDescriptor().setShardId(shardId);
+        return;
       }
       try {
         Thread.sleep(1000);
@@ -1222,7 +1275,7 @@ public final class ZkController {
     }
     
     throw new SolrException(ErrorCode.SERVER_ERROR,
-        "Could not get shard_id for core: " + coreName + " coreNodeName:" + coreNodeName);
+        "Could not get shard id for core: " + cd.getName());
   }
   
   public static void uploadToZK(SolrZkClient zkClient, File dir, String zkPath) throws IOException, KeeperException, InterruptedException {
@@ -1261,7 +1314,7 @@ public final class ZkController {
   
   public String getCoreNodeName(CoreDescriptor descriptor){
     String coreNodeName = descriptor.getCloudDescriptor().getCoreNodeName();
-    if (coreNodeName == null) {
+    if (coreNodeName == null && !genericCoreNodeNames) {
       // it's the default
       return getNodeName() + "_" + descriptor.getName();
     }
@@ -1277,27 +1330,33 @@ public final class ZkController {
     downloadFromZK(zkClient, ZkController.CONFIGS_ZKNODE + "/" + configName, dir);
   }
 
-  public void preRegister(CoreDescriptor cd) throws KeeperException, InterruptedException {
-
-
-    // before becoming available, make sure we are not live and active
-    // this also gets us our assigned shard id if it was not specified
-    publish(cd, ZkStateReader.DOWN, false);
-
-    String coreNodeName = getCoreNodeName(cd);
+  public void preRegister(CoreDescriptor cd ) {
     
+    String coreNodeName = getCoreNodeName(cd);
+
     // make sure the node name is set on the descriptor
     if (cd.getCloudDescriptor().getCoreNodeName() == null) {
       cd.getCloudDescriptor().setCoreNodeName(coreNodeName);
     }
+
+    // before becoming available, make sure we are not live and active
+    // this also gets us our assigned shard id if it was not specified
+    try {
+      publish(cd, ZkStateReader.DOWN, false);
+    } catch (KeeperException e) {
+      log.error("", e);
+      throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "", e);
+    } catch (InterruptedException e) {
+      Thread.currentThread().interrupt();
+      log.error("", e);
+      throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "", e);
+    }
     
     if (cd.getCloudDescriptor().getShardId() == null && needsToBeAssignedShardId(cd, zkStateReader.getClusterState(), coreNodeName)) {
-      String shardId;
-      shardId = doGetShardIdProcess(cd.getName(), cd);
-      cd.getCloudDescriptor().setShardId(shardId);
+      doGetShardIdAndNodeNameProcess(cd);
     } else {
       // still wait till we see us in local state
-      doGetShardIdProcess(cd.getName(), cd);
+      doGetShardIdAndNodeNameProcess(cd);
     }
 
   }
@@ -1332,7 +1391,7 @@ public final class ZkController {
         }
       }
     }
-    
+
     String leaderBaseUrl = leaderProps.getBaseUrl();
     String leaderCoreName = leaderProps.getCoreName();
     
@@ -1494,11 +1553,11 @@ public final class ZkController {
   }
   
   /**
-   * utilitiy method fro trimming and leading and/or trailing slashes from 
+   * Utility method for trimming and leading and/or trailing slashes from 
    * it's input.  May return the empty string.  May return null if and only 
    * if the input is null.
    */
-  private static String trimLeadingAndTrailingSlashes(final String in) {
+  public static String trimLeadingAndTrailingSlashes(final String in) {
     if (null == in) return in;
     
     String out = in;

Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/CachingDirectoryFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/CachingDirectoryFactory.java?rev=1498702&r1=1498701&r2=1498702&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/CachingDirectoryFactory.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/CachingDirectoryFactory.java Mon Jul  1 21:31:19 2013
@@ -32,6 +32,7 @@ import java.util.Set;
 import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext.Context;
+import org.apache.lucene.store.NRTCachingDirectory;
 import org.apache.lucene.store.NativeFSLockFactory;
 import org.apache.lucene.store.NoLockFactory;
 import org.apache.lucene.store.RateLimitedDirectoryWrapper;
@@ -40,6 +41,9 @@ import org.apache.lucene.store.SingleIns
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.SolrException.ErrorCode;
 import org.apache.solr.common.util.NamedList;
+import org.apache.solr.store.blockcache.BlockDirectory;
+import org.apache.solr.store.hdfs.HdfsDirectory;
+import org.apache.solr.store.hdfs.HdfsLockFactory;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -498,6 +502,24 @@ public abstract class CachingDirectoryFa
     } else if ("single".equals(lockType)) {
       if (!(dir.getLockFactory() instanceof SingleInstanceLockFactory)) dir
           .setLockFactory(new SingleInstanceLockFactory());
+    } else if ("hdfs".equals(lockType)) {
+      Directory del = dir;
+      
+      if (dir instanceof NRTCachingDirectory) {
+        del = ((NRTCachingDirectory) del).getDelegate();
+      }
+      
+      if (del instanceof BlockDirectory) {
+        del = ((BlockDirectory) del).getDirectory();
+      }
+      
+      if (!(del instanceof HdfsDirectory)) {
+        throw new SolrException(ErrorCode.FORBIDDEN, "Directory: "
+            + del.getClass().getName()
+            + ", but hdfs lock factory can only be used with HdfsDirectory");
+      }
+
+      dir.setLockFactory(new HdfsLockFactory(((HdfsDirectory)del).getHdfsDirPath(), ((HdfsDirectory)del).getConfiguration()));
     } else if ("none".equals(lockType)) {
       // Recipe for disaster
       log.error("CONFIGURATION WARNING: locks are disabled on " + dir);
@@ -519,7 +541,7 @@ public abstract class CachingDirectoryFa
     return path;
   }
   
-  private String stripTrailingSlash(String path) {
+  protected String stripTrailingSlash(String path) {
     if (path.endsWith("/")) {
       path = path.substring(0, path.length() - 1);
     }

Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/ConfigSolr.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/ConfigSolr.java?rev=1498702&r1=1498701&r2=1498702&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/ConfigSolr.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/ConfigSolr.java Mon Jul  1 21:31:19 2013
@@ -63,6 +63,7 @@ public abstract class ConfigSolr {
     SOLR_SHARDHANDLERFACTORY_SOCKETTIMEOUT,
     SOLR_SHARESCHEMA,
     SOLR_TRANSIENTCACHESIZE,
+    SOLR_GENERICCORENODENAMES,
     SOLR_ZKCLIENTTIMEOUT,
     SOLR_ZKHOST,
 

Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/ConfigSolrXml.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/ConfigSolrXml.java?rev=1498702&r1=1498701&r2=1498702&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/ConfigSolrXml.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/ConfigSolrXml.java Mon Jul  1 21:31:19 2013
@@ -77,6 +77,7 @@ public class ConfigSolrXml extends Confi
     failIfFound("solr/cores/@hostContext");
     failIfFound("solr/cores/@hostPort");
     failIfFound("solr/cores/@leaderVoteWait");
+    failIfFound("solr/cores/@genericCoreNodeNames");
     failIfFound("solr/cores/@managementPath");
     failIfFound("solr/cores/@shareSchema");
     failIfFound("solr/cores/@transientCacheSize");
@@ -118,6 +119,7 @@ public class ConfigSolrXml extends Confi
     propMap.put(CfgProp.SOLR_HOSTCONTEXT, doSub("solr/solrcloud/str[@name='hostContext']"));
     propMap.put(CfgProp.SOLR_HOSTPORT, doSub("solr/solrcloud/int[@name='hostPort']"));
     propMap.put(CfgProp.SOLR_LEADERVOTEWAIT, doSub("solr/solrcloud/int[@name='leaderVoteWait']"));
+    propMap.put(CfgProp.SOLR_GENERICCORENODENAMES, doSub("solr/solrcloud/bool[@name='genericCoreNodeNames']"));
     propMap.put(CfgProp.SOLR_MANAGEMENTPATH, doSub("solr/str[@name='managementPath']"));
     propMap.put(CfgProp.SOLR_SHAREDLIB, doSub("solr/str[@name='sharedLib']"));
     propMap.put(CfgProp.SOLR_SHARESCHEMA, doSub("solr/str[@name='shareSchema']"));

Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/ConfigSolrXmlOld.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/ConfigSolrXmlOld.java?rev=1498702&r1=1498701&r2=1498702&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/ConfigSolrXmlOld.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/ConfigSolrXmlOld.java Mon Jul  1 21:31:19 2013
@@ -73,6 +73,7 @@ public class ConfigSolrXmlOld extends Co
     failIfFound("solr/solrcloud/str[@name='hostContext']");
     failIfFound("solr/solrcloud/int[@name='hostPort']");
     failIfFound("solr/solrcloud/int[@name='leaderVoteWait']");
+    failIfFound("solr/solrcloud/int[@name='genericCoreNodeNames']");
     failIfFound("solr/str[@name='managementPath']");
     failIfFound("solr/str[@name='sharedLib']");
     failIfFound("solr/str[@name='shareSchema']");
@@ -125,6 +126,8 @@ public class ConfigSolrXmlOld extends Co
         config.getVal("solr/cores/@hostPort", false));
     propMap.put(CfgProp.SOLR_LEADERVOTEWAIT,
         config.getVal("solr/cores/@leaderVoteWait", false));
+    propMap.put(CfgProp.SOLR_GENERICCORENODENAMES,
+        config.getVal("solr/cores/@genericCoreNodeNames", false));
     propMap.put(CfgProp.SOLR_MANAGEMENTPATH,
         config.getVal("solr/cores/@managementPath", false));
     propMap.put(CfgProp.SOLR_SHARESCHEMA,

Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/CoreContainer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/CoreContainer.java?rev=1498702&r1=1498701&r2=1498702&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/CoreContainer.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/CoreContainer.java Mon Jul  1 21:31:19 2013
@@ -182,7 +182,7 @@ public class CoreContainer
       cores = new CoreContainer(solrHome);
 
       // first we find zkhost, then we check for solr.xml in zk
-      // 1. look for zkhost from sys prop 2. look for zkhost in {solr.home}/solr.properties
+      // 1. look for zkhost from sys prop 2. TODO: look for zkhost in {solr.home}/solr.properties
       
       // Either we have a config file or not.
       
@@ -317,13 +317,15 @@ public class CoreContainer
 
     transientCacheSize = cfg.getInt(ConfigSolr.CfgProp.SOLR_TRANSIENTCACHESIZE, Integer.MAX_VALUE);
 
+    boolean genericCoreNodeNames = cfg.getBool(ConfigSolr.CfgProp.SOLR_GENERICCORENODENAMES, false);
+
     if (shareSchema) {
       indexSchemaCache = new ConcurrentHashMap<String,IndexSchema>();
     }
 
     zkClientTimeout = Integer.parseInt(System.getProperty("zkClientTimeout",
         Integer.toString(zkClientTimeout)));
-    zkSys.initZooKeeper(this, solrHome, zkHost, zkClientTimeout, hostPort, hostContext, host, leaderVoteWait, distribUpdateConnTimeout, distribUpdateSoTimeout);
+    zkSys.initZooKeeper(this, solrHome, zkHost, zkClientTimeout, hostPort, hostContext, host, leaderVoteWait, genericCoreNodeNames, distribUpdateConnTimeout, distribUpdateSoTimeout);
     
     if (isZooKeeperAware() && coreLoadThreads <= 1) {
       throw new SolrException(ErrorCode.SERVER_ERROR,
@@ -1144,6 +1146,8 @@ public class CoreContainer
         zkSys.getHostContext());
     addAttrib(coresAttribs, ConfigSolr.CfgProp.SOLR_LEADERVOTEWAIT, "leaderVoteWait",
         zkSys.getLeaderVoteWait(), LEADER_VOTE_WAIT);
+    addAttrib(coresAttribs, ConfigSolr.CfgProp.SOLR_GENERICCORENODENAMES, "genericCoreNodeNames",
+        Boolean.toString(zkSys.getGenericCoreNodeNames()), "false");
     if (transientCacheSize != Integer.MAX_VALUE) { // This test
     // is a consequence of testing. I really hate it.
       addAttrib(coresAttribs, ConfigSolr.CfgProp.SOLR_TRANSIENTCACHESIZE, "transientCacheSize",
@@ -1221,18 +1225,7 @@ public class CoreContainer
   }
 
   public void preRegisterInZk(final CoreDescriptor p) {
-    try {
-      zkSys.getZkController().preRegister(p);
-    } catch (KeeperException e) {
-      log.error("", e);
-      throw new ZooKeeperException(
-          SolrException.ErrorCode.SERVER_ERROR, "", e);
-    } catch (InterruptedException e) {
-      Thread.currentThread().interrupt();
-      log.error("", e);
-      throw new ZooKeeperException(
-          SolrException.ErrorCode.SERVER_ERROR, "", e);
-    }
+    zkSys.getZkController().preRegister(p);
   }
 
   public String getSolrHome() {

Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java?rev=1498702&r1=1498701&r2=1498702&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java Mon Jul  1 21:31:19 2013
@@ -167,7 +167,6 @@ public abstract class DirectoryFactory i
    */
   public abstract void release(Directory directory) throws IOException;
   
-  
   /**
    * Normalize a given path.
    * 
@@ -229,5 +228,21 @@ public abstract class DirectoryFactory i
     }
     return isSuccess;
   }
-  
+
+  /**
+   * If your implementation can count on delete-on-last-close semantics
+   * or throws an exception when trying to remove a file in use, return
+   * false (eg NFS). Otherwise, return true. Defaults to returning false.
+   * 
+   * @return true if factory impl requires that Searcher's explicitly
+   * reserve commit points.
+   */
+  public boolean searchersReserveCommitPoints() {
+    return false;
+  }
+
+  public String getDataHome(CoreDescriptor cd) throws IOException {
+    // by default, we go off the instance directory
+    return normalize(SolrResourceLoader.normalizeDir(cd.getInstanceDir()) + cd.getDataDir());
+  }
 }

Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/EphemeralDirectoryFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/EphemeralDirectoryFactory.java?rev=1498702&r1=1498701&r2=1498702&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/EphemeralDirectoryFactory.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/EphemeralDirectoryFactory.java Mon Jul  1 21:31:19 2013
@@ -51,4 +51,16 @@ public abstract class EphemeralDirectory
   public boolean isAbsolute(String path) {
     return true;
   }
+  
+  
+  @Override
+  public void remove(Directory dir) throws IOException {
+    // ram dir does not persist its dir anywhere
+  }
+  
+  @Override
+  public void remove(String path) throws IOException {
+    // ram dir does not persist its dir anywhere
+  }
+
 }

Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/SolrCore.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/SolrCore.java?rev=1498702&r1=1498701&r2=1498702&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/SolrCore.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/SolrCore.java Mon Jul  1 21:31:19 2013
@@ -673,11 +673,10 @@ public final class SolrCore implements S
     if (dataDir == null) {
       if (cd.usingDefaultDataDir()) dataDir = config.getDataDir();
       if (dataDir == null) {
-        dataDir = cd.getDataDir();
         try {
+          dataDir = cd.getDataDir();
           if (!directoryFactory.isAbsolute(dataDir)) {
-            dataDir = directoryFactory.normalize(SolrResourceLoader
-                .normalizeDir(cd.getInstanceDir()) + dataDir);
+            dataDir = directoryFactory.getDataHome(cd);
           }
         } catch (IOException e) {
           throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, null, e);
@@ -748,7 +747,6 @@ public final class SolrCore implements S
       this.codec = initCodec(solrConfig, schema);
       
       if (updateHandler == null) {
-        initDirectoryFactory();
         solrCoreState = new DefaultSolrCoreState(getDirectoryFactory());
       } else {
         solrCoreState = updateHandler.getSolrCoreState();

Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/ZkContainer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/ZkContainer.java?rev=1498702&r1=1498701&r2=1498702&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/ZkContainer.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/ZkContainer.java Mon Jul  1 21:31:19 2013
@@ -55,6 +55,7 @@ public class ZkContainer {
   private String hostContext;
   private String host;
   private String leaderVoteWait;
+  private Boolean genericCoreNodeNames;
   private int distribUpdateConnTimeout;
 
   public SolrZkServer getZkServer() {
@@ -80,6 +81,10 @@ public class ZkContainer {
   public String getLeaderVoteWait() {
     return leaderVoteWait;
   }
+  
+  public boolean getGenericCoreNodeNames() {
+    return genericCoreNodeNames;
+  }
 
   public int getDistribUpdateConnTimeout() {
     return distribUpdateConnTimeout;
@@ -95,7 +100,7 @@ public class ZkContainer {
     
   }
   
-  public void initZooKeeper(final CoreContainer cc, String solrHome, String zkHost, int zkClientTimeout, String hostPort, String hostContext, String host, String leaderVoteWait, int distribUpdateConnTimeout, int distribUpdateSoTimeout) {
+  public void initZooKeeper(final CoreContainer cc, String solrHome, String zkHost, int zkClientTimeout, String hostPort, String hostContext, String host, String leaderVoteWait, boolean genericCoreNodeNames, int distribUpdateConnTimeout, int distribUpdateSoTimeout) {
     ZkController zkController = null;
     
     // if zkHost sys property is not set, we are not using ZooKeeper
@@ -113,6 +118,7 @@ public class ZkContainer {
     this.hostContext = hostContext;
     this.host = host;
     this.leaderVoteWait = leaderVoteWait;
+    this.genericCoreNodeNames = genericCoreNodeNames;
     this.distribUpdateConnTimeout = distribUpdateConnTimeout;
     this.distribUpdateSoTimeout = distribUpdateSoTimeout;
     
@@ -175,7 +181,7 @@ public class ZkContainer {
         }
         zkController = new ZkController(cc, zookeeperHost, zkClientTimeout,
             zkClientConnectTimeout, host, hostPort, hostContext,
-            leaderVoteWait, distribUpdateConnTimeout, distribUpdateSoTimeout,
+            leaderVoteWait, genericCoreNodeNames, distribUpdateConnTimeout, distribUpdateSoTimeout,
             new CurrentCoreDescriptorProvider() {
 
               @Override

Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java?rev=1498702&r1=1498701&r2=1498702&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java Mon Jul  1 21:31:19 2013
@@ -362,20 +362,18 @@ public class HttpShardHandler extends Sh
             Map<String, Replica> sliceShards = slice.getReplicasMap();
 
             // For now, recreate the | delimited list of equivalent servers
-            Set<String> liveNodes = clusterState.getLiveNodes();
             StringBuilder sliceShardsStr = new StringBuilder();
             boolean first = true;
-            for (ZkNodeProps nodeProps : sliceShards.values()) {
-              ZkCoreNodeProps coreNodeProps = new ZkCoreNodeProps(nodeProps);
-              if (!liveNodes.contains(coreNodeProps.getNodeName())
-                  || !coreNodeProps.getState().equals(
+            for (Replica replica : sliceShards.values()) {
+              if (!clusterState.liveNodesContain(replica.getNodeName())
+                  || !replica.getStr(ZkStateReader.STATE_PROP).equals(
                       ZkStateReader.ACTIVE)) continue;
               if (first) {
                 first = false;
               } else {
                 sliceShardsStr.append('|');
               }
-              String url = coreNodeProps.getCoreUrl();
+              String url = ZkCoreNodeProps.getCoreUrl(replica);
               if (url.startsWith("http://"))
                 url = url.substring(7);
               sliceShardsStr.append(url);

Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java?rev=1498702&r1=1498701&r2=1498702&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java Mon Jul  1 21:31:19 2013
@@ -192,7 +192,13 @@ public class SolrIndexSearcher extends I
     this.name = "Searcher@" + Integer.toHexString(hashCode()) + (name!=null ? " "+name : "");
     log.info("Opening " + this.name);
 
-    Directory dir = this.reader.directory();
+    if (directoryFactory.searchersReserveCommitPoints()) {
+      // reserve commit point for life of searcher
+      core.getDeletionPolicy().saveCommitPoint(
+          reader.getIndexCommit().getGeneration());
+    }
+    
+    Directory dir = getIndexReader().directory();
     
     this.reserveDirectory = reserveDirectory;
     this.createdDirectory = r == null;
@@ -330,12 +336,18 @@ public class SolrIndexSearcher extends I
     // super.close();
     // can't use super.close() since it just calls reader.close() and that may only be called once
     // per reader (even if incRef() was previously called).
+    
+    long cpg = reader.getIndexCommit().getGeneration();
     try {
       if (closeReader) reader.decRef();
     } catch (Throwable t) {
       SolrException.log(log, "Problem dec ref'ing reader", t);
     }
 
+    if (directoryFactory.searchersReserveCommitPoints()) {
+      core.getDeletionPolicy().releaseCommitPoint(cpg);
+    }
+
     for (SolrCache cache : cacheList) {
       cache.close();
     }

Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/ReusedBufferedIndexOutput.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/ReusedBufferedIndexOutput.java?rev=1498702&r1=1497072&r2=1498702&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/ReusedBufferedIndexOutput.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/ReusedBufferedIndexOutput.java Mon Jul  1 21:31:19 2013
@@ -92,6 +92,23 @@ public abstract class ReusedBufferedInde
   protected abstract void seekInternal(long pos) throws IOException;
   
   @Override
+  public void seek(long pos) throws IOException {
+    if (pos > fileLength) {
+      fileLength = pos;
+    }
+
+    if (pos >= bufferStart && pos < (bufferStart + bufferLength))
+      bufferPosition = (int)(pos - bufferStart);  // seek within buffer
+    else {
+      flushBufferToCache();
+      bufferStart = pos;
+      bufferPosition = 0;
+      bufferLength = 0;
+      seekInternal(pos);
+    }
+  }
+  
+  @Override
   public long length() throws IOException {
     return fileLength;
   }

Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/hdfs/HdfsDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/hdfs/HdfsDirectory.java?rev=1498702&r1=1497072&r2=1498702&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/hdfs/HdfsDirectory.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/hdfs/HdfsDirectory.java Mon Jul  1 21:31:19 2013
@@ -244,6 +244,11 @@ public class HdfsDirectory extends Direc
     }
     
     @Override
+    public void seek(long pos) throws IOException {
+      writer.seek(pos);
+    }
+    
+    @Override
     public void writeByte(byte b) throws IOException {
       writer.writeByte(b);
     }

Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/hdfs/NullIndexOutput.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/hdfs/NullIndexOutput.java?rev=1498702&r1=1497072&r2=1498702&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/hdfs/NullIndexOutput.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/hdfs/NullIndexOutput.java Mon Jul  1 21:31:19 2013
@@ -46,6 +46,12 @@ public class NullIndexOutput extends Ind
     return length;
   }
   
+
+  @Override
+  public void seek(long pos) throws IOException {
+    this.pos = pos;
+  }
+  
   @Override
   public void writeByte(byte b) throws IOException {
     pos++;



Mime
View raw message