lucene-commits mailing list archives

Site index · List index

Message view	« Date » · « Thread »
Top	« Date » · « Thread »
From	markrmil...@apache.org
Subject	svn commit: r1498702 [1/2] - in /lucene/dev/branches/branch_4x: ./ lucene/ lucene/tools/ lucene/tools/junit4/ solr/ solr/core/ solr/core/src/java/org/apache/solr/ solr/core/src/java/org/apache/solr/client/solrj/embedded/ solr/core/src/java/org/apache/s...
Date	Mon, 01 Jul 2013 21:31:21 GMT
Author: markrmiller Date: Mon Jul 1 21:31:19 2013 New Revision: 1498702 URL: http://svn.apache.org/r1498702 Log: SOLR-4916: Add support to write and read Solr index files and transaction log files to and from HDFS. Added: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/Assign.java - copied unchanged from r1497072, lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/Assign.java lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java - copied unchanged from r1497072, lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/ - copied from r1497072, lucene/dev/trunk/solr/core/src/java/org/apache/solr/store/ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/HdfsTransactionLog.java - copied unchanged from r1497072, lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/HdfsTransactionLog.java lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/HdfsUpdateLog.java - copied unchanged from r1497072, lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/HdfsUpdateLog.java lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/util/HdfsUtil.java - copied unchanged from r1497072, lucene/dev/trunk/solr/core/src/java/org/apache/solr/util/HdfsUtil.java lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/util/IOUtils.java - copied unchanged from r1497072, lucene/dev/trunk/solr/core/src/java/org/apache/solr/util/IOUtils.java lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/AssignTest.java - copied unchanged from r1497072, lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/AssignTest.java lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/hdfs/ - copied from r1497072, lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/hdfs/ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java - copied unchanged from r1497072, lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/store/ - copied from r1497072, lucene/dev/trunk/solr/core/src/test/org/apache/solr/store/ lucene/dev/branches/branch_4x/solr/example/alt-configs/ - copied from r1497072, lucene/dev/trunk/solr/example/alt-configs/ lucene/dev/branches/branch_4x/solr/licenses/commons-configuration-1.6.jar.sha1 - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/commons-configuration-1.6.jar.sha1 lucene/dev/branches/branch_4x/solr/licenses/commons-configuration-LICENSE-ASL.txt - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/commons-configuration-LICENSE-ASL.txt lucene/dev/branches/branch_4x/solr/licenses/commons-configuration-NOTICE.txt - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/commons-configuration-NOTICE.txt lucene/dev/branches/branch_4x/solr/licenses/concurrentlinkedhashmap-lru-1.2.jar.sha1 - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/concurrentlinkedhashmap-lru-1.2.jar.sha1 lucene/dev/branches/branch_4x/solr/licenses/concurrentlinkedhashmap-lru-LICENSE-ASL.txt - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/concurrentlinkedhashmap-lru-LICENSE-ASL.txt lucene/dev/branches/branch_4x/solr/licenses/concurrentlinkedhashmap-lru-NOTICE.txt - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/concurrentlinkedhashmap-lru-NOTICE.txt lucene/dev/branches/branch_4x/solr/licenses/hadoop-annotations-2.0.5-alpha.jar.sha1 - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/hadoop-annotations-2.0.5-alpha.jar.sha1 lucene/dev/branches/branch_4x/solr/licenses/hadoop-annotations-LICENSE-ASL.txt - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/hadoop-annotations-LICENSE-ASL.txt lucene/dev/branches/branch_4x/solr/licenses/hadoop-annotations-NOTICE.txt - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/hadoop-annotations-NOTICE.txt lucene/dev/branches/branch_4x/solr/licenses/hadoop-auth-2.0.5-alpha.jar.sha1 - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/hadoop-auth-2.0.5-alpha.jar.sha1 lucene/dev/branches/branch_4x/solr/licenses/hadoop-auth-LICENSE-ASL.txt - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/hadoop-auth-LICENSE-ASL.txt lucene/dev/branches/branch_4x/solr/licenses/hadoop-auth-NOTICE.txt - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/hadoop-auth-NOTICE.txt lucene/dev/branches/branch_4x/solr/licenses/hadoop-common-2.0.0-cdh4.2.0-SNAPSHOT-tests-NOTICE.txt - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/hadoop-common-2.0.0-cdh4.2.0-SNAPSHOT-tests-NOTICE.txt lucene/dev/branches/branch_4x/solr/licenses/hadoop-common-2.0.5-alpha-tests.jar.sha1 - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/hadoop-common-2.0.5-alpha-tests.jar.sha1 lucene/dev/branches/branch_4x/solr/licenses/hadoop-common-2.0.5-alpha.jar.sha1 - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/hadoop-common-2.0.5-alpha.jar.sha1 lucene/dev/branches/branch_4x/solr/licenses/hadoop-common-LICENSE-ASL.txt - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/hadoop-common-LICENSE-ASL.txt lucene/dev/branches/branch_4x/solr/licenses/hadoop-common-NOTICE.txt - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/hadoop-common-NOTICE.txt lucene/dev/branches/branch_4x/solr/licenses/hadoop-common-tests-LICENSE-ASL.txt - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/hadoop-common-tests-LICENSE-ASL.txt lucene/dev/branches/branch_4x/solr/licenses/hadoop-hdfs-2.0.5-alpha-tests.jar.sha1 - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/hadoop-hdfs-2.0.5-alpha-tests.jar.sha1 lucene/dev/branches/branch_4x/solr/licenses/hadoop-hdfs-2.0.5-alpha.jar.sha1 - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/hadoop-hdfs-2.0.5-alpha.jar.sha1 lucene/dev/branches/branch_4x/solr/licenses/hadoop-hdfs-LICENSE-ASL.txt - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/hadoop-hdfs-LICENSE-ASL.txt lucene/dev/branches/branch_4x/solr/licenses/hadoop-hdfs-NOTICE.txt - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/hadoop-hdfs-NOTICE.txt lucene/dev/branches/branch_4x/solr/licenses/hadoop-hdfs-tests-LICENSE-ASL.txt - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/hadoop-hdfs-tests-LICENSE-ASL.txt lucene/dev/branches/branch_4x/solr/licenses/hadoop-hdfs-tests-NOTICE.txt - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/hadoop-hdfs-tests-NOTICE.txt lucene/dev/branches/branch_4x/solr/licenses/jersey-core-1.16.jar.sha1 - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/jersey-core-1.16.jar.sha1 lucene/dev/branches/branch_4x/solr/licenses/jersey-core-LICENSE-CDDL.txt - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/jersey-core-LICENSE-CDDL.txt lucene/dev/branches/branch_4x/solr/licenses/jetty-6.1.26.jar.sha1 - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/jetty-6.1.26.jar.sha1 lucene/dev/branches/branch_4x/solr/licenses/jetty-util-6.1.26.jar.sha1 - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/jetty-util-6.1.26.jar.sha1 lucene/dev/branches/branch_4x/solr/licenses/log4j-1.2.17.jar.sha1 - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/log4j-1.2.17.jar.sha1 lucene/dev/branches/branch_4x/solr/licenses/protobuf-java-2.4.0a.jar.sha1 - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/protobuf-java-2.4.0a.jar.sha1 lucene/dev/branches/branch_4x/solr/licenses/protobuf-java-LICENSE-BSD.txt - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/protobuf-java-LICENSE-BSD.txt lucene/dev/branches/branch_4x/solr/licenses/protobuf-java-NOTICE.txt - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/protobuf-java-NOTICE.txt lucene/dev/branches/branch_4x/solr/licenses/zookeeper-3.4.5.jar.sha1 - copied unchanged from r1497072, lucene/dev/trunk/solr/licenses/zookeeper-3.4.5.jar.sha1 Removed: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/AssignShard.java Modified: lucene/dev/branches/branch_4x/ (props changed) lucene/dev/branches/branch_4x/.gitignore lucene/dev/branches/branch_4x/build.xml lucene/dev/branches/branch_4x/lucene/ (props changed) lucene/dev/branches/branch_4x/lucene/common-build.xml (contents, props changed) lucene/dev/branches/branch_4x/lucene/tools/ (props changed) lucene/dev/branches/branch_4x/lucene/tools/custom-tasks.xml lucene/dev/branches/branch_4x/lucene/tools/junit4/tests.policy lucene/dev/branches/branch_4x/solr/ (props changed) lucene/dev/branches/branch_4x/solr/CHANGES.txt lucene/dev/branches/branch_4x/solr/NOTICE.txt (contents, props changed) lucene/dev/branches/branch_4x/solr/build.xml (contents, props changed) lucene/dev/branches/branch_4x/solr/common-build.xml (contents, props changed) lucene/dev/branches/branch_4x/solr/core/ (props changed) lucene/dev/branches/branch_4x/solr/core/ivy.xml lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/SolrLogFormatter.java lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/CloudDescriptor.java lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/Overseer.java lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/ZkController.java lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/CachingDirectoryFactory.java lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/ConfigSolr.java lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/ConfigSolrXml.java lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/ConfigSolrXmlOld.java lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/CoreContainer.java lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/EphemeralDirectoryFactory.java lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/SolrCore.java lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/ZkContainer.java lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/ReusedBufferedIndexOutput.java lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/hdfs/HdfsDirectory.java lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/hdfs/NullIndexOutput.java lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/TransactionLog.java lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/UpdateHandler.java lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/UpdateLog.java lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java lucene/dev/branches/branch_4x/solr/core/src/test-files/log4j.properties lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/solr-no-core.xml lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/solr.xml lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ClusterStateUpdateTest.java lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/search/TestRecovery.java lucene/dev/branches/branch_4x/solr/example/ (props changed) lucene/dev/branches/branch_4x/solr/example/resources/log4j.properties lucene/dev/branches/branch_4x/solr/example/solr/solr.xml lucene/dev/branches/branch_4x/solr/licenses/ (props changed) lucene/dev/branches/branch_4x/solr/solrj/ (props changed) lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java lucene/dev/branches/branch_4x/solr/solrj/src/test-files/solrj/log4j.properties lucene/dev/branches/branch_4x/solr/solrj/src/test-files/solrj/solr/shared/solr.xml lucene/dev/branches/branch_4x/solr/solrj/src/test-files/solrj/solr/solr.xml lucene/dev/branches/branch_4x/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/TestSolrProperties.java lucene/dev/branches/branch_4x/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestCoreAdmin.java lucene/dev/branches/branch_4x/solr/test-framework/ (props changed) lucene/dev/branches/branch_4x/solr/test-framework/ivy.xml lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDistribZkTestBase.java lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/util/TestHarness.java Modified: lucene/dev/branches/branch_4x/.gitignore URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/.gitignore?rev=1498702&r1=1498701&r2=1498702&view=diff ============================================================================== --- lucene/dev/branches/branch_4x/.gitignore (original) +++ lucene/dev/branches/branch_4x/.gitignore Mon Jul 1 21:31:19 2013 @@ -1,4 +1,5 @@ - +# hdfs +/solr/example/hdfs .jar # . Modified: lucene/dev/branches/branch_4x/build.xml URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/build.xml?rev=1498702&r1=1498701&r2=1498702&view=diff ============================================================================== --- lucene/dev/branches/branch_4x/build.xml (original) +++ lucene/dev/branches/branch_4x/build.xml Mon Jul 1 21:31:19 2013 @@ -205,7 +205,7 @@ <!-- TODO: find a better way to exclude duplicate JAR files & fix the servlet-api mess! --> <pathconvert property="eclipse.fileset.libs" pathsep="\|" dirsep="/"> <fileset dir="${basedir}/lucene" includes="/lib/.jar" excludes="*/servlet-api.jar, analysis/uima/, tools/, build/"/> - <fileset dir="${basedir}/solr" includes="/lib/.jar" excludes="core/lib/servlet-api.jar, contrib/analysis-extras/, test-framework/, build/, dist/, package/" /> + <fileset dir="${basedir}/solr" includes="/lib/.jar" excludes="core/lib/servlet-api.jar, contrib/analysis-extras/, test-framework/lib/junit, test-framework/lib/ant, test-framework/lib/randomizedtesting, build/, dist/, package/*" /> <map from="${basedir}/" to=""/> </pathconvert> <xslt in="${ant.file}" out=".classpath" style="dev-tools/eclipse/dot.classpath.xsl" force="true"> Modified: lucene/dev/branches/branch_4x/lucene/common-build.xml URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/common-build.xml?rev=1498702&r1=1498701&r2=1498702&view=diff ============================================================================== --- lucene/dev/branches/branch_4x/lucene/common-build.xml (original) +++ lucene/dev/branches/branch_4x/lucene/common-build.xml Mon Jul 1 21:31:19 2013 @@ -359,7 +359,7 @@ <target name="resolve" depends="ivy-availability-check,ivy-configure"> <!-- todo, make this a property or something. only special cases need bundles --> - <ivy:retrieve type="jar,bundle" log="download-only" + <ivy:retrieve type="jar,bundle,tests" log="download-only" conf="${ivy.default.configuration}" sync="${ivy.sync}"/> </target> Modified: lucene/dev/branches/branch_4x/lucene/tools/custom-tasks.xml URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/tools/custom-tasks.xml?rev=1498702&r1=1498701&r2=1498702&view=diff ============================================================================== --- lucene/dev/branches/branch_4x/lucene/tools/custom-tasks.xml (original) +++ lucene/dev/branches/branch_4x/lucene/tools/custom-tasks.xml Mon Jul 1 21:31:19 2013 @@ -43,7 +43,7 @@ <!-- Typical version patterns. --> <replaceregex pattern="\.rc[0-9]+" replace="" flags="gi" /> - <replaceregex pattern="\-(r)?([0-9\-\_\.])+(b(eta)?([0-9\-\.]))?$" replace="" flags="gi" /> + <replaceregex pattern="\-(r)?([0-9\-\_\.])+((b(eta)?)\|((a(lpha)?))([0-9\-\.]))?(\-tests)?$" replace="" flags="gi" /> <!-- git hashcode pattern: its always 40 chars right? --> <replaceregex pattern="\-[a-z0-9]{40,40}$" replace="" flags="gi" /> Modified: lucene/dev/branches/branch_4x/lucene/tools/junit4/tests.policy URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/tools/junit4/tests.policy?rev=1498702&r1=1498701&r2=1498702&view=diff ============================================================================== --- lucene/dev/branches/branch_4x/lucene/tools/junit4/tests.policy (original) +++ lucene/dev/branches/branch_4x/lucene/tools/junit4/tests.policy Mon Jul 1 21:31:19 2013 @@ -59,6 +59,10 @@ grant { permission javax.management.MBeanPermission "", ""; permission javax.management.MBeanServerPermission ""; permission javax.management.MBeanTrustPermission ""; + permission javax.security.auth.AuthPermission ""; + permission javax.security.auth.PrivateCredentialPermission "org.apache.hadoop.security.Credentials * \"\"", "read"; + permission java.security.SecurityPermission "putProviderProperty.SaslPlainServer"; + permission java.security.SecurityPermission "insertProvider.SaslPlainServer"; // TIKA uses BouncyCastle and that registers new provider for PDF parsing + MSOffice parsing. Maybe report as bug! permission java.security.SecurityPermission "putProviderProperty.BC"; Modified: lucene/dev/branches/branch_4x/solr/CHANGES.txt URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/CHANGES.txt?rev=1498702&r1=1498701&r2=1498702&view=diff ============================================================================== --- lucene/dev/branches/branch_4x/solr/CHANGES.txt (original) +++ lucene/dev/branches/branch_4x/solr/CHANGES.txt Mon Jul 1 21:31:19 2013 @@ -113,6 +113,9 @@ New Features SOLR-4897: Add solr/example/example-schemaless/, an example config set for schemaless mode. (Steve Rowe) +* SOLR-4916: Add support to write and read Solr index files and transaction log + files to and from HDFS. (phunt, Mark Miller, Greg Chanan) + Bug Fixes ---------------------- Modified: lucene/dev/branches/branch_4x/solr/NOTICE.txt URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/NOTICE.txt?rev=1498702&r1=1498701&r2=1498702&view=diff ============================================================================== --- lucene/dev/branches/branch_4x/solr/NOTICE.txt (original) +++ lucene/dev/branches/branch_4x/solr/NOTICE.txt Mon Jul 1 21:31:19 2013 @@ -88,6 +88,7 @@ including, but not limited to: - Apache Jakarta Regexp - Apache Commons - Apache Xerces + - Apache Blur ICU4J, (under analysis/icu) is licensed under an MIT styles license and Copyright (c) 1995-2008 International Business Machines Corporation and others @@ -108,6 +109,9 @@ http://bitbucket.org/jpbarrette/moman/ov The class org.apache.lucene.util.WeakIdentityMap was derived from the Apache CXF project and is Apache License 2.0. +The HdfsDirectory and BlockDirectory were derived from +the Apache Blur incubating project and are Apache License 2.0. + The Google Code Prettify is Apache License 2.0. See http://code.google.com/p/google-code-prettify/ Modified: lucene/dev/branches/branch_4x/solr/build.xml URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/build.xml?rev=1498702&r1=1498701&r2=1498702&view=diff ============================================================================== --- lucene/dev/branches/branch_4x/solr/build.xml (original) +++ lucene/dev/branches/branch_4x/solr/build.xml Mon Jul 1 21:31:19 2013 @@ -40,7 +40,7 @@ <!-- ========================================================================= --> <target name="example" description="Creates a runnable example configuration." - depends="dist-contrib,dist-war"> + depends="dist-contrib,dist-war,setup-alt-examples"> <copy file="${dist}/${fullnamever}.war" tofile="${example}/webapps/${ant.project.name}.war"/> <jar destfile="${example}/exampledocs/post.jar" @@ -56,6 +56,15 @@ <echo>See ${example}/README.txt for how to run the Solr example configuration.</echo> </target> + <target name="setup-alt-examples"> + <copy todir="${example}/hdfs" overwrite="true"> + <fileset dir="${example}/solr"/> + </copy> + <copy todir="${example}/hdfs/collection1/conf" overwrite="true"> + <fileset dir="${example}/alt-configs/hdfs"/> + </copy> + </target> + <target name="run-example" depends="example" description="Run Solr interactively, via Jetty. -Dexample.debug=true to enable JVM debugger"> <property name="example.solr.home" location="example/solr"/> Modified: lucene/dev/branches/branch_4x/solr/common-build.xml URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/common-build.xml?rev=1498702&r1=1498701&r2=1498702&view=diff ============================================================================== --- lucene/dev/branches/branch_4x/solr/common-build.xml (original) +++ lucene/dev/branches/branch_4x/solr/common-build.xml Mon Jul 1 21:31:19 2013 @@ -96,6 +96,12 @@ <path id="solr.test.base.classpath"> <pathelement path="${common-solr.dir}/build/solr-test-framework/classes/java"/> + <fileset dir="${common-solr.dir}/test-framework/lib"> + <include name=".jar"/> + <exclude name="junit-.jar" /> + <exclude name="randomizedtesting-runner-.jar" /> + <exclude name="ant.jar" /> + </fileset> <pathelement path="${build.dir}/test-files"/> <path refid="test.base.classpath"/> </path> Modified: lucene/dev/branches/branch_4x/solr/core/ivy.xml URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/ivy.xml?rev=1498702&r1=1498701&r2=1498702&view=diff ============================================================================== --- lucene/dev/branches/branch_4x/solr/core/ivy.xml (original) +++ lucene/dev/branches/branch_4x/solr/core/ivy.xml Mon Jul 1 21:31:19 2013 @@ -16,6 +16,9 @@ specific language governing permissions and limitations under the License. --> +<!DOCTYPE ivy-module [ + <!ENTITY hadoop.version "2.0.5-alpha"> +]> <ivy-module version="2.0"> <info organisation="org.apache.solr" module="core"/> @@ -33,6 +36,14 @@ <dependency org="org.restlet.jee" name="org.restlet" rev="2.1.1" transitive="false"/> <dependency org="org.restlet.jee" name="org.restlet.ext.servlet" rev="2.1.1" transitive="false"/> <dependency org="joda-time" name="joda-time" rev="2.2" transitive="false"/> - <exclude org="" ext="" matcher="regexp" type="${ivy.exclude.types}"/> + + <dependency org="org.apache.hadoop" name="hadoop-common" rev="&hadoop.version;" transitive="false"/> + <dependency org="org.apache.hadoop" name="hadoop-hdfs" rev="&hadoop.version;" transitive="false"/> + <dependency org="org.apache.hadoop" name="hadoop-annotations" rev="&hadoop.version;" transitive="false"/> + <dependency org="org.apache.hadoop" name="hadoop-auth" rev="&hadoop.version;" transitive="false"/> + <dependency org="commons-configuration" name="commons-configuration" rev="1.6" transitive="false"/> + <dependency org="com.google.protobuf" name="protobuf-java" rev="2.4.0a" transitive="false"/> + <dependency org="com.googlecode.concurrentlinkedhashmap" name="concurrentlinkedhashmap-lru" rev="1.2" transitive="false"/> + <exclude org="" ext="" matcher="regexp" type="${ivy.exclude.types}"/> </dependencies> </ivy-module> Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/SolrLogFormatter.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/SolrLogFormatter.java?rev=1498702&r1=1498701&r2=1498702&view=diff ============================================================================== --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/SolrLogFormatter.java (original) +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/SolrLogFormatter.java Mon Jul 1 21:31:19 2013 @@ -17,21 +17,25 @@ package org.apache.solr; * limitations under the License. / +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.WeakHashMap; +import java.util.logging.ConsoleHandler; +import java.util.logging.Formatter; +import java.util.logging.Handler; +import java.util.logging.Level; +import java.util.logging.LogRecord; +import java.util.logging.Logger; + import org.apache.solr.cloud.ZkController; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.core.SolrCore; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrRequestInfo; import org.slf4j.LoggerFactory; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.WeakHashMap; -import java.util.logging.; - public class SolrLogFormatter extends Formatter { /** Add this interface to a thread group and the string returned by @@ -259,7 +263,7 @@ sb.append("(group_name=").append(tg.getN private Map<String,Object> getReplicaProps(ZkController zkController, SolrCore core) { final String collection = core.getCoreDescriptor().getCloudDescriptor().getCollectionName(); - Replica replica = zkController.getClusterState().getReplica(collection, zkController.getCoreNodeName(core.getCoreDescriptor())); + Replica replica = zkController.getClusterState().getReplica(collection, core.getCoreDescriptor().getCloudDescriptor().getCoreNodeName()); if(replica!=null) { return replica.getProperties(); } Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java?rev=1498702&r1=1498701&r2=1498702&view=diff ============================================================================== --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java (original) +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java Mon Jul 1 21:31:19 2013 @@ -85,6 +85,7 @@ public class JettySolrRunner { private String shards; private String dataDir; + private String solrUlogDir; private volatile boolean startedBefore = false; @@ -359,6 +360,9 @@ public class JettySolrRunner { if( dataDir != null) { System.setProperty("solr.data.dir", dataDir); } + if( solrUlogDir != null) { + System.setProperty("solr.ulog.dir", solrUlogDir); + } if(shards != null) { System.setProperty("shard", shards); } @@ -382,6 +386,8 @@ public class JettySolrRunner { System.clearProperty("shard"); System.clearProperty("solr.data.dir"); System.clearProperty("coreNodeName"); + System.clearProperty("solr.ulog.dir"); + } public void stop() throws Exception { @@ -485,6 +491,10 @@ public class JettySolrRunner { public void setDataDir(String dataDir) { this.dataDir = dataDir; } + + public void setUlogDir(String ulogDir) { + this.solrUlogDir = ulogDir; + } public void setCoreNodeName(String coreNodeName) { this.coreNodeName = coreNodeName; Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/CloudDescriptor.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/CloudDescriptor.java?rev=1498702&r1=1498701&r2=1498702&view=diff ============================================================================== --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/CloudDescriptor.java (original) +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/CloudDescriptor.java Mon Jul 1 21:31:19 2013 @@ -44,6 +44,10 @@ public class CloudDescriptor { public boolean isLeader() { return isLeader; } + + public void setLeader(boolean isLeader) { + this.isLeader = isLeader; + } public void setShardId(String shardId) { this.shardId = shardId; Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java?rev=1498702&r1=1498701&r2=1498702&view=diff ============================================================================== --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java (original) +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java Mon Jul 1 21:31:19 2013 @@ -47,9 +47,9 @@ public abstract class ElectionContext { String leaderSeqPath; private SolrZkClient zkClient; - public ElectionContext(final String shardZkNodeName, + public ElectionContext(final String coreNodeName, final String electionPath, final String leaderPath, final ZkNodeProps leaderProps, final SolrZkClient zkClient) { - this.id = shardZkNodeName; + this.id = coreNodeName; this.electionPath = electionPath; this.leaderPath = leaderPath; this.leaderProps = leaderProps; @@ -78,8 +78,8 @@ class ShardLeaderElectionContextBase ext protected LeaderElector leaderElector; public ShardLeaderElectionContextBase(LeaderElector leaderElector, final String shardId, - final String collection, final String shardZkNodeName, ZkNodeProps props, ZkStateReader zkStateReader) { - super(shardZkNodeName, ZkStateReader.COLLECTIONS_ZKNODE + "/" + collection + "/leader_elect/" + final String collection, final String coreNodeName, ZkNodeProps props, ZkStateReader zkStateReader) { + super(coreNodeName, ZkStateReader.COLLECTIONS_ZKNODE + "/" + collection + "/leader_elect/" + shardId, ZkStateReader.getShardLeadersPath(collection, shardId), props, zkStateReader.getZkClient()); this.leaderElector = leaderElector; @@ -95,7 +95,7 @@ class ShardLeaderElectionContextBase ext zkClient.makePath(leaderPath, ZkStateReader.toJSON(leaderProps), CreateMode.EPHEMERAL, true); assert shardId != null; - ZkNodeProps m = ZkNodeProps.fromKeyVals(Overseer.QUEUE_OPERATION, "leader", + ZkNodeProps m = ZkNodeProps.fromKeyVals(Overseer.QUEUE_OPERATION, ZkStateReader.LEADER_PROP, ZkStateReader.SHARD_ID_PROP, shardId, ZkStateReader.COLLECTION_PROP, collection, ZkStateReader.BASE_URL_PROP, leaderProps.getProperties() .get(ZkStateReader.BASE_URL_PROP), ZkStateReader.CORE_NAME_PROP, @@ -119,8 +119,8 @@ final class ShardLeaderElectionContext e public ShardLeaderElectionContext(LeaderElector leaderElector, final String shardId, final String collection, - final String shardZkNodeName, ZkNodeProps props, ZkController zkController, CoreContainer cc) { - super(leaderElector, shardId, collection, shardZkNodeName, props, + final String coreNodeName, ZkNodeProps props, ZkController zkController, CoreContainer cc) { + super(leaderElector, shardId, collection, coreNodeName, props, zkController.getZkStateReader()); this.zkController = zkController; this.cc = cc; @@ -138,12 +138,12 @@ final class ShardLeaderElectionContext e @Override void runLeaderProcess(boolean weAreReplacement) throws KeeperException, InterruptedException, IOException { - log.info("Running the leader process."); + log.info("Running the leader process for shard " + shardId); String coreName = leaderProps.getStr(ZkStateReader.CORE_NAME_PROP); // clear the leader in clusterstate - ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION, "leader", + ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION, ZkStateReader.LEADER_PROP, ZkStateReader.SHARD_ID_PROP, shardId, ZkStateReader.COLLECTION_PROP, collection); Overseer.getInQueue(zkClient).offer(ZkStateReader.toJSON(m)); @@ -243,8 +243,8 @@ final class ShardLeaderElectionContext e } log.info("I am the new leader: " - + ZkCoreNodeProps.getCoreUrl(leaderProps)); - core.getCoreDescriptor().getCloudDescriptor().isLeader = true; + + ZkCoreNodeProps.getCoreUrl(leaderProps) + " " + shardId); + core.getCoreDescriptor().getCloudDescriptor().setLeader(true); } finally { if (core != null) { core.close(); @@ -254,16 +254,17 @@ final class ShardLeaderElectionContext e try { super.runLeaderProcess(weAreReplacement); } catch (Throwable t) { + SolrException.log(log, "There was a problem trying to register as the leader", t); + cancelElection(); try { core = cc.getCore(coreName); if (core == null) { - cancelElection(); throw new SolrException(ErrorCode.SERVER_ERROR, "Fatal Error, SolrCore not found:" + coreName + " in " + cc.getCoreNames()); } - core.getCoreDescriptor().getCloudDescriptor().isLeader = false; + core.getCoreDescriptor().getCloudDescriptor().setLeader(false); // we could not publish ourselves as leader - rejoin election rejoinLeaderElection(leaderSeqPath, core); @@ -332,7 +333,7 @@ final class ShardLeaderElectionContext e return; } else { if (cnt % 40 == 0) { - log.info("Waiting until we see more replicas up: total=" + log.info("Waiting until we see more replicas up for shard " + shardId + ": total=" + slices.getReplicasMap().size() + " found=" + found + " timeoutin=" + (timeoutAt - System.currentTimeMillis())); } Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/Overseer.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/Overseer.java?rev=1498702&r1=1498701&r2=1498702&view=diff ============================================================================== --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/Overseer.java (original) +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/Overseer.java Mon Jul 1 21:31:19 2013 @@ -50,15 +50,16 @@ import org.slf4j.LoggerFactory; / public class Overseer { public static final String QUEUE_OPERATION = "operation"; + public static final String DELETECORE = "deletecore"; public static final String REMOVECOLLECTION = "removecollection"; private static final int STATE_UPDATE_DELAY = 1500; // delay between cloud state updates + private static Logger log = LoggerFactory.getLogger(Overseer.class); private class ClusterStateUpdater implements Runnable, ClosableThread { - private static final String DELETECORE = "deletecore"; private final ZkStateReader reader; private final SolrZkClient zkClient; private final String myId; @@ -267,8 +268,14 @@ public class Overseer { final String collection = message.getStr(ZkStateReader.COLLECTION_PROP); String coreNodeName = message.getStr(ZkStateReader.CORE_NODE_NAME_PROP); if (coreNodeName == null) { - // it must be the default then - coreNodeName = message.getStr(ZkStateReader.NODE_NAME_PROP) + "_" + message.getStr(ZkStateReader.CORE_NAME_PROP); + coreNodeName = getAssignedCoreNodeName(state, message); + if (coreNodeName != null) { + log.info("node=" + coreNodeName + " is already registered"); + } else { + // if coreNodeName is null, auto assign one + coreNodeName = Assign.assignNode(collection, state); + } + message.getProperties().put(ZkStateReader.CORE_NODE_NAME_PROP, coreNodeName); } Integer numShards = message.getStr(ZkStateReader.NUM_SHARDS_PROP)!=null?Integer.parseInt(message.getStr(ZkStateReader.NUM_SHARDS_PROP)):null; log.info("Update state numShards={} message={}", numShards, message); @@ -281,7 +288,6 @@ public class Overseer { // use the provided non null shardId String sliceName = message.getStr(ZkStateReader.SHARD_ID_PROP); if (sliceName == null) { - //String nodeName = message.getStr(ZkStateReader.NODE_NAME_PROP); //get shardId from ClusterState sliceName = getAssignedId(state, coreNodeName, message); if (sliceName != null) { @@ -295,8 +301,8 @@ public class Overseer { numShards = state.getCollectionStates().get(collection).getSlices().size(); log.info("Collection already exists with " + ZkStateReader.NUM_SHARDS_PROP + "=" + numShards); } - sliceName = AssignShard.assignShard(collection, state, numShards); - log.info("Assigning new node to shard=" + sliceName); + sliceName = Assign.assignShard(collection, state, numShards); + log.info("Assigning new node to shard shard=" + sliceName); } Slice slice = state.getSlice(collection, sliceName); @@ -320,8 +326,11 @@ public class Overseer { } } - // we don't put num_shards in the clusterstate + // we don't put these in the clusterstate replicaProps.remove(ZkStateReader.NUM_SHARDS_PROP); + replicaProps.remove(ZkStateReader.CORE_NODE_NAME_PROP); + replicaProps.remove(ZkStateReader.SHARD_ID_PROP); + replicaProps.remove(ZkStateReader.COLLECTION_PROP); replicaProps.remove(QUEUE_OPERATION); // remove any props with null values @@ -417,6 +426,26 @@ public class Overseer { return null; } + private String getAssignedCoreNodeName(ClusterState state, ZkNodeProps message) { + Collection<Slice> slices = state.getSlices(message.getStr(ZkStateReader.COLLECTION_PROP)); + if (slices != null) { + for (Slice slice : slices) { + for (Replica replica : slice.getReplicas()) { + String baseUrl = replica.getStr(ZkStateReader.BASE_URL_PROP); + String core = replica.getStr(ZkStateReader.CORE_NAME_PROP); + + String msgBaseUrl = message.getStr(ZkStateReader.BASE_URL_PROP); + String msgCore = message.getStr(ZkStateReader.CORE_NAME_PROP); + + if (baseUrl.equals(msgBaseUrl) && core.equals(msgCore)) { + return replica.getName(); + } + } + } + } + return null; + } + private ClusterState updateSlice(ClusterState state, String collectionName, Slice slice) { // System.out.println("###!!!### OLD CLUSTERSTATE: " + JSONUtil.toJSON(state.getCollectionStates())); // System.out.println("Updating slice:" + slice); @@ -526,10 +555,6 @@ public class Overseer { private ClusterState removeCore(final ClusterState clusterState, ZkNodeProps message) { String cnn = message.getStr(ZkStateReader.CORE_NODE_NAME_PROP); - if (cnn == null) { - // it must be the default then - cnn = message.getStr(ZkStateReader.NODE_NAME_PROP) + "_" + message.getStr(ZkStateReader.CORE_NAME_PROP); - } final String collection = message.getStr(ZkStateReader.COLLECTION_PROP); Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java?rev=1498702&r1=1498701&r2=1498702&view=diff ============================================================================== --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java (original) +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java Mon Jul 1 21:31:19 2013 @@ -400,10 +400,11 @@ public class OverseerCollectionProcessor // wait for parent leader to acknowledge the sub-shard core log.info("Asking parent leader to wait for: " + subShardName + " to be alive on: " + nodeName); + String coreNodeName = waitForCoreNodeName(collection, zkStateReader.getZkClient().getBaseUrlForNodeName(nodeName), subShardName); CoreAdminRequest.WaitForState cmd = new CoreAdminRequest.WaitForState(); cmd.setCoreName(subShardName); cmd.setNodeName(nodeName); - cmd.setCoreNodeName(nodeName + "_" + subShardName); + cmd.setCoreNodeName(coreNodeName); cmd.setState(ZkStateReader.ACTIVE); cmd.setCheckLive(true); cmd.setOnlyIfLeader(true); @@ -506,12 +507,13 @@ public class OverseerCollectionProcessor sendShardRequest(subShardNodeName, params); + String coreNodeName = waitForCoreNodeName(collection, zkStateReader.getZkClient().getBaseUrlForNodeName(subShardNodeName), shardName); // wait for the replicas to be seen as active on sub shard leader log.info("Asking sub shard leader to wait for: " + shardName + " to be alive on: " + subShardNodeName); CoreAdminRequest.WaitForState cmd = new CoreAdminRequest.WaitForState(); cmd.setCoreName(subShardNames.get(i-1)); cmd.setNodeName(subShardNodeName); - cmd.setCoreNodeName(subShardNodeName + "_" + shardName); + cmd.setCoreNodeName(coreNodeName); cmd.setState(ZkStateReader.ACTIVE); cmd.setCheckLive(true); cmd.setOnlyIfLeader(true); @@ -545,6 +547,35 @@ public class OverseerCollectionProcessor throw new SolrException(ErrorCode.SERVER_ERROR, null, e); } } + + private String waitForCoreNodeName(DocCollection collection, String msgBaseUrl, String msgCore) { + int retryCount = 320; + while (retryCount-- > 0) { + Map<String,Slice> slicesMap = zkStateReader.getClusterState() + .getSlicesMap(collection.getName()); + if (slicesMap != null) { + + for (Slice slice : slicesMap.values()) { + for (Replica replica : slice.getReplicas()) { + // TODO: for really large clusters, we could 'index' on this + + String baseUrl = replica.getStr(ZkStateReader.BASE_URL_PROP); + String core = replica.getStr(ZkStateReader.CORE_NAME_PROP); + + if (baseUrl.equals(msgBaseUrl) && core.equals(msgCore)) { + return replica.getName(); + } + } + } + } + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + throw new SolrException(ErrorCode.SERVER_ERROR, "Could not find coreNodeName"); + } private void collectShardResponses(NamedList results, boolean abortOnError, String msgOnError) { ShardResponse srsp; Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java?rev=1498702&r1=1498701&r2=1498702&view=diff ============================================================================== --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java (original) +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java Mon Jul 1 21:31:19 2013 @@ -91,7 +91,7 @@ public class RecoveryStrategy extends Th zkController = cc.getZkController(); zkStateReader = zkController.getZkStateReader(); baseUrl = zkController.getBaseUrl(); - coreZkNodeName = zkController.getCoreNodeName(cd); + coreZkNodeName = cd.getCloudDescriptor().getCoreNodeName(); } public void setRecoveringAfterStartup(boolean recoveringAfterStartup) { @@ -325,10 +325,10 @@ public class RecoveryStrategy extends Th String ourUrl = ZkCoreNodeProps.getCoreUrl(baseUrl, coreName); boolean isLeader = leaderUrl.equals(ourUrl); - if (isLeader && !cloudDesc.isLeader) { + if (isLeader && !cloudDesc.isLeader()) { throw new SolrException(ErrorCode.SERVER_ERROR, "Cloud state still says we are leader."); } - if (cloudDesc.isLeader) { + if (cloudDesc.isLeader()) { // we are now the leader - no one else must have been suitable log.warn("We have not yet recovered - but we are now the leader! core=" + coreName); log.info("Finished recovery process. core=" + coreName); Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java?rev=1498702&r1=1498701&r2=1498702&view=diff ============================================================================== --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java (original) +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java Mon Jul 1 21:31:19 2013 @@ -146,8 +146,7 @@ public class SyncStrategy { private boolean syncWithReplicas(ZkController zkController, SolrCore core, ZkNodeProps props, String collection, String shardId) { List<ZkCoreNodeProps> nodes = zkController.getZkStateReader() - .getReplicaProps(collection, shardId, - zkController.getCoreNodeName(core.getCoreDescriptor()), + .getReplicaProps(collection, shardId,core.getCoreDescriptor().getCloudDescriptor().getCoreNodeName(), props.getStr(ZkStateReader.CORE_NAME_PROP)); if (nodes == null) { @@ -175,7 +174,7 @@ public class SyncStrategy { List<ZkCoreNodeProps> nodes = zkController .getZkStateReader() .getReplicaProps(collection, shardId, - zkController.getCoreNodeName(cd), + cd.getCloudDescriptor().getCoreNodeName(), leaderProps.getStr(ZkStateReader.CORE_NAME_PROP)); if (nodes == null) { log.info(ZkCoreNodeProps.getCoreUrl(leaderProps) + " has no replicas"); Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/ZkController.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/ZkController.java?rev=1498702&r1=1498701&r2=1498702&view=diff ============================================================================== --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/ZkController.java (original) +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/ZkController.java Mon Jul 1 21:31:19 2013 @@ -132,6 +132,8 @@ public final class ZkController { protected volatile Overseer overseer; private String leaderVoteWait; + + private boolean genericCoreNodeNames; private int clientTimeout; @@ -140,11 +142,11 @@ public final class ZkController { private UpdateShardHandler updateShardHandler; public ZkController(final CoreContainer cc, String zkServerAddress, int zkClientTimeout, int zkClientConnectTimeout, String localHost, String locaHostPort, - String localHostContext, String leaderVoteWait, int distribUpdateConnTimeout, int distribUpdateSoTimeout, final CurrentCoreDescriptorProvider registerOnReconnect) throws InterruptedException, + String localHostContext, String leaderVoteWait, boolean genericCoreNodeNames, int distribUpdateConnTimeout, int distribUpdateSoTimeout, final CurrentCoreDescriptorProvider registerOnReconnect) throws InterruptedException, TimeoutException, IOException { if (cc == null) throw new IllegalArgumentException("CoreContainer cannot be null."); this.cc = cc; - + this.genericCoreNodeNames = genericCoreNodeNames; // be forgiving and strip this off leading/trailing slashes // this allows us to support users specifying hostContext="/" in // solr.xml to indicate the root context, instead of hostContext="" @@ -254,9 +256,9 @@ public final class ZkController { // before registering as live, make sure everyone is in a // down state for (CoreDescriptor descriptor : descriptors) { - final String coreZkNodeName = getCoreNodeName(descriptor); + final String coreZkNodeName = descriptor.getCloudDescriptor().getCoreNodeName(); try { - descriptor.getCloudDescriptor().isLeader = false; + descriptor.getCloudDescriptor().setLeader(false); publish(descriptor, ZkStateReader.DOWN, updateLastPublished); } catch (Exception e) { if (isClosed) { @@ -323,7 +325,7 @@ public final class ZkController { .getCurrentDescriptors(); if (descriptors != null) { for (CoreDescriptor descriptor : descriptors) { - descriptor.getCloudDescriptor().isLeader = false; + descriptor.getCloudDescriptor().setLeader(false); } } } @@ -544,7 +546,6 @@ public final class ZkController { if (replica.getNodeName().equals(getNodeName()) && !(replica.getStr(ZkStateReader.STATE_PROP) .equals(ZkStateReader.DOWN))) { - assert replica.getStr(ZkStateReader.SHARD_ID_PROP) != null; ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION, "state", ZkStateReader.STATE_PROP, ZkStateReader.DOWN, ZkStateReader.BASE_URL_PROP, getBaseUrl(), @@ -555,8 +556,7 @@ public final class ZkController { ZkStateReader.NODE_NAME_PROP, getNodeName(), ZkStateReader.SHARD_ID_PROP, replica.getStr(ZkStateReader.SHARD_ID_PROP), - ZkStateReader.COLLECTION_PROP, - replica.getStr(ZkStateReader.COLLECTION_PROP), + ZkStateReader.COLLECTION_PROP, collectionName, ZkStateReader.CORE_NODE_NAME_PROP, replica.getName()); updatedNodes.add(replica.getStr(ZkStateReader.CORE_NAME_PROP)); overseerJobQueue.offer(ZkStateReader.toJSON(m)); @@ -735,7 +735,8 @@ public final class ZkController { final CloudDescriptor cloudDesc = desc.getCloudDescriptor(); final String collection = cloudDesc.getCollectionName(); - final String coreZkNodeName = getCoreNodeName(desc); + final String coreZkNodeName = desc.getCloudDescriptor().getCoreNodeName(); + assert coreZkNodeName != null : "we should have a coreNodeName by now"; String shardId = cloudDesc.getShardId(); @@ -923,16 +924,16 @@ public final class ZkController { props.put(ZkStateReader.CORE_NAME_PROP, cd.getName()); props.put(ZkStateReader.NODE_NAME_PROP, getNodeName()); - final String coreZkNodeName = getCoreNodeName(cd); + final String coreNodeName = cd.getCloudDescriptor().getCoreNodeName(); ZkNodeProps ourProps = new ZkNodeProps(props); String collection = cd.getCloudDescriptor() .getCollectionName(); ElectionContext context = new ShardLeaderElectionContext(leaderElector, shardId, - collection, coreZkNodeName, ourProps, this, cc); + collection, coreNodeName, ourProps, this, cc); leaderElector.setup(context); - electionContexts.put(coreZkNodeName, context); + electionContexts.put(coreNodeName, context); leaderElector.joinElection(context, false); } @@ -1017,7 +1018,7 @@ public final class ZkController { final CloudDescriptor cloudDesc = desc.getCloudDescriptor(); - final String shardId = state.getShardId(coreNodeName); + final String shardId = state.getShardId(getBaseUrl(), desc.getName()); if (shardId != null) { cloudDesc.setShardId(shardId); @@ -1028,16 +1029,21 @@ public final class ZkController { public void unregister(String coreName, CoreDescriptor cd) throws InterruptedException, KeeperException { - final String zkNodeName = getCoreNodeName(cd); - ElectionContext context = electionContexts.remove(zkNodeName); + final String coreNodeName = cd.getCloudDescriptor().getCoreNodeName(); + ElectionContext context = electionContexts.remove(coreNodeName); + + assert context != null : coreNodeName; + if (context != null) { context.cancelElection(); } + CloudDescriptor cloudDescriptor = cd.getCloudDescriptor(); ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION, - "deletecore", ZkStateReader.CORE_NAME_PROP, coreName, + Overseer.DELETECORE, ZkStateReader.CORE_NAME_PROP, coreName, ZkStateReader.NODE_NAME_PROP, getNodeName(), - ZkStateReader.COLLECTION_PROP, cd.getCloudDescriptor().getCollectionName()); + ZkStateReader.COLLECTION_PROP, cloudDescriptor.getCollectionName(), + ZkStateReader.CORE_NODE_NAME_PROP, coreNodeName); overseerJobQueue.offer(ZkStateReader.toJSON(m)); } @@ -1206,13 +1212,60 @@ public final class ZkController { return zkStateReader; } - private String doGetShardIdProcess(String coreName, CoreDescriptor descriptor) { - final String coreNodeName = getCoreNodeName(descriptor); + private void doGetShardIdAndNodeNameProcess(CoreDescriptor cd) { + final String coreNodeName = cd.getCloudDescriptor().getCoreNodeName(); + + if (coreNodeName != null) { + waitForShardId(cd); + } else { + // if no explicit coreNodeName, we want to match by base url and core name + waitForCoreNodeName(cd); + waitForShardId(cd); + } + } + + private void waitForCoreNodeName(CoreDescriptor descriptor) { int retryCount = 320; + log.info("look for our core node name"); while (retryCount-- > 0) { - final String shardId = zkStateReader.getClusterState().getShardId(coreNodeName); + Map<String,Slice> slicesMap = zkStateReader.getClusterState() + .getSlicesMap(descriptor.getCloudDescriptor().getCollectionName()); + if (slicesMap != null) { + + for (Slice slice : slicesMap.values()) { + for (Replica replica : slice.getReplicas()) { + // TODO: for really large clusters, we could 'index' on this + + String baseUrl = replica.getStr(ZkStateReader.BASE_URL_PROP); + String core = replica.getStr(ZkStateReader.CORE_NAME_PROP); + + String msgBaseUrl = getBaseUrl(); + String msgCore = descriptor.getName(); + + if (baseUrl.equals(msgBaseUrl) && core.equals(msgCore)) { + descriptor.getCloudDescriptor() + .setCoreNodeName(replica.getName()); + return; + } + } + } + } + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + } + + private void waitForShardId(CoreDescriptor cd) { + log.info("waiting to find shard id in clusterstate for " + cd.getName()); + int retryCount = 320; + while (retryCount-- > 0) { + final String shardId = zkStateReader.getClusterState().getShardId(getBaseUrl(), cd.getName()); if (shardId != null) { - return shardId; + cd.getCloudDescriptor().setShardId(shardId); + return; } try { Thread.sleep(1000); @@ -1222,7 +1275,7 @@ public final class ZkController { } throw new SolrException(ErrorCode.SERVER_ERROR, - "Could not get shard_id for core: " + coreName + " coreNodeName:" + coreNodeName); + "Could not get shard id for core: " + cd.getName()); } public static void uploadToZK(SolrZkClient zkClient, File dir, String zkPath) throws IOException, KeeperException, InterruptedException { @@ -1261,7 +1314,7 @@ public final class ZkController { public String getCoreNodeName(CoreDescriptor descriptor){ String coreNodeName = descriptor.getCloudDescriptor().getCoreNodeName(); - if (coreNodeName == null) { + if (coreNodeName == null && !genericCoreNodeNames) { // it's the default return getNodeName() + "_" + descriptor.getName(); } @@ -1277,27 +1330,33 @@ public final class ZkController { downloadFromZK(zkClient, ZkController.CONFIGS_ZKNODE + "/" + configName, dir); } - public void preRegister(CoreDescriptor cd) throws KeeperException, InterruptedException { - - - // before becoming available, make sure we are not live and active - // this also gets us our assigned shard id if it was not specified - publish(cd, ZkStateReader.DOWN, false); - - String coreNodeName = getCoreNodeName(cd); + public void preRegister(CoreDescriptor cd ) { + String coreNodeName = getCoreNodeName(cd); + // make sure the node name is set on the descriptor if (cd.getCloudDescriptor().getCoreNodeName() == null) { cd.getCloudDescriptor().setCoreNodeName(coreNodeName); } + + // before becoming available, make sure we are not live and active + // this also gets us our assigned shard id if it was not specified + try { + publish(cd, ZkStateReader.DOWN, false); + } catch (KeeperException e) { + log.error("", e); + throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "", e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + log.error("", e); + throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "", e); + } if (cd.getCloudDescriptor().getShardId() == null && needsToBeAssignedShardId(cd, zkStateReader.getClusterState(), coreNodeName)) { - String shardId; - shardId = doGetShardIdProcess(cd.getName(), cd); - cd.getCloudDescriptor().setShardId(shardId); + doGetShardIdAndNodeNameProcess(cd); } else { // still wait till we see us in local state - doGetShardIdProcess(cd.getName(), cd); + doGetShardIdAndNodeNameProcess(cd); } } @@ -1332,7 +1391,7 @@ public final class ZkController { } } } - + String leaderBaseUrl = leaderProps.getBaseUrl(); String leaderCoreName = leaderProps.getCoreName(); @@ -1494,11 +1553,11 @@ public final class ZkController { } /* - * utilitiy method fro trimming and leading and/or trailing slashes from + * Utility method for trimming and leading and/or trailing slashes from * it's input. May return the empty string. May return null if and only * if the input is null. / - private static String trimLeadingAndTrailingSlashes(final String in) { + public static String trimLeadingAndTrailingSlashes(final String in) { if (null == in) return in; String out = in; Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/CachingDirectoryFactory.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/CachingDirectoryFactory.java?rev=1498702&r1=1498701&r2=1498702&view=diff ============================================================================== --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/CachingDirectoryFactory.java (original) +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/CachingDirectoryFactory.java Mon Jul 1 21:31:19 2013 @@ -32,6 +32,7 @@ import java.util.Set; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext.Context; +import org.apache.lucene.store.NRTCachingDirectory; import org.apache.lucene.store.NativeFSLockFactory; import org.apache.lucene.store.NoLockFactory; import org.apache.lucene.store.RateLimitedDirectoryWrapper; @@ -40,6 +41,9 @@ import org.apache.lucene.store.SingleIns import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.util.NamedList; +import org.apache.solr.store.blockcache.BlockDirectory; +import org.apache.solr.store.hdfs.HdfsDirectory; +import org.apache.solr.store.hdfs.HdfsLockFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -498,6 +502,24 @@ public abstract class CachingDirectoryFa } else if ("single".equals(lockType)) { if (!(dir.getLockFactory() instanceof SingleInstanceLockFactory)) dir .setLockFactory(new SingleInstanceLockFactory()); + } else if ("hdfs".equals(lockType)) { + Directory del = dir; + + if (dir instanceof NRTCachingDirectory) { + del = ((NRTCachingDirectory) del).getDelegate(); + } + + if (del instanceof BlockDirectory) { + del = ((BlockDirectory) del).getDirectory(); + } + + if (!(del instanceof HdfsDirectory)) { + throw new SolrException(ErrorCode.FORBIDDEN, "Directory: " + + del.getClass().getName() + + ", but hdfs lock factory can only be used with HdfsDirectory"); + } + + dir.setLockFactory(new HdfsLockFactory(((HdfsDirectory)del).getHdfsDirPath(), ((HdfsDirectory)del).getConfiguration())); } else if ("none".equals(lockType)) { // Recipe for disaster log.error("CONFIGURATION WARNING: locks are disabled on " + dir); @@ -519,7 +541,7 @@ public abstract class CachingDirectoryFa return path; } - private String stripTrailingSlash(String path) { + protected String stripTrailingSlash(String path) { if (path.endsWith("/")) { path = path.substring(0, path.length() - 1); } Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/ConfigSolr.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/ConfigSolr.java?rev=1498702&r1=1498701&r2=1498702&view=diff ============================================================================== --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/ConfigSolr.java (original) +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/ConfigSolr.java Mon Jul 1 21:31:19 2013 @@ -63,6 +63,7 @@ public abstract class ConfigSolr { SOLR_SHARDHANDLERFACTORY_SOCKETTIMEOUT, SOLR_SHARESCHEMA, SOLR_TRANSIENTCACHESIZE, + SOLR_GENERICCORENODENAMES, SOLR_ZKCLIENTTIMEOUT, SOLR_ZKHOST, Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/ConfigSolrXml.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/ConfigSolrXml.java?rev=1498702&r1=1498701&r2=1498702&view=diff ============================================================================== --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/ConfigSolrXml.java (original) +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/ConfigSolrXml.java Mon Jul 1 21:31:19 2013 @@ -77,6 +77,7 @@ public class ConfigSolrXml extends Confi failIfFound("solr/cores/@hostContext"); failIfFound("solr/cores/@hostPort"); failIfFound("solr/cores/@leaderVoteWait"); + failIfFound("solr/cores/@genericCoreNodeNames"); failIfFound("solr/cores/@managementPath"); failIfFound("solr/cores/@shareSchema"); failIfFound("solr/cores/@transientCacheSize"); @@ -118,6 +119,7 @@ public class ConfigSolrXml extends Confi propMap.put(CfgProp.SOLR_HOSTCONTEXT, doSub("solr/solrcloud/str[@name='hostContext']")); propMap.put(CfgProp.SOLR_HOSTPORT, doSub("solr/solrcloud/int[@name='hostPort']")); propMap.put(CfgProp.SOLR_LEADERVOTEWAIT, doSub("solr/solrcloud/int[@name='leaderVoteWait']")); + propMap.put(CfgProp.SOLR_GENERICCORENODENAMES, doSub("solr/solrcloud/bool[@name='genericCoreNodeNames']")); propMap.put(CfgProp.SOLR_MANAGEMENTPATH, doSub("solr/str[@name='managementPath']")); propMap.put(CfgProp.SOLR_SHAREDLIB, doSub("solr/str[@name='sharedLib']")); propMap.put(CfgProp.SOLR_SHARESCHEMA, doSub("solr/str[@name='shareSchema']")); Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/ConfigSolrXmlOld.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/ConfigSolrXmlOld.java?rev=1498702&r1=1498701&r2=1498702&view=diff ============================================================================== --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/ConfigSolrXmlOld.java (original) +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/ConfigSolrXmlOld.java Mon Jul 1 21:31:19 2013 @@ -73,6 +73,7 @@ public class ConfigSolrXmlOld extends Co failIfFound("solr/solrcloud/str[@name='hostContext']"); failIfFound("solr/solrcloud/int[@name='hostPort']"); failIfFound("solr/solrcloud/int[@name='leaderVoteWait']"); + failIfFound("solr/solrcloud/int[@name='genericCoreNodeNames']"); failIfFound("solr/str[@name='managementPath']"); failIfFound("solr/str[@name='sharedLib']"); failIfFound("solr/str[@name='shareSchema']"); @@ -125,6 +126,8 @@ public class ConfigSolrXmlOld extends Co config.getVal("solr/cores/@hostPort", false)); propMap.put(CfgProp.SOLR_LEADERVOTEWAIT, config.getVal("solr/cores/@leaderVoteWait", false)); + propMap.put(CfgProp.SOLR_GENERICCORENODENAMES, + config.getVal("solr/cores/@genericCoreNodeNames", false)); propMap.put(CfgProp.SOLR_MANAGEMENTPATH, config.getVal("solr/cores/@managementPath", false)); propMap.put(CfgProp.SOLR_SHARESCHEMA, Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/CoreContainer.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/CoreContainer.java?rev=1498702&r1=1498701&r2=1498702&view=diff ============================================================================== --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/CoreContainer.java (original) +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/CoreContainer.java Mon Jul 1 21:31:19 2013 @@ -182,7 +182,7 @@ public class CoreContainer cores = new CoreContainer(solrHome); // first we find zkhost, then we check for solr.xml in zk - // 1. look for zkhost from sys prop 2. look for zkhost in {solr.home}/solr.properties + // 1. look for zkhost from sys prop 2. TODO: look for zkhost in {solr.home}/solr.properties // Either we have a config file or not. @@ -317,13 +317,15 @@ public class CoreContainer transientCacheSize = cfg.getInt(ConfigSolr.CfgProp.SOLR_TRANSIENTCACHESIZE, Integer.MAX_VALUE); + boolean genericCoreNodeNames = cfg.getBool(ConfigSolr.CfgProp.SOLR_GENERICCORENODENAMES, false); + if (shareSchema) { indexSchemaCache = new ConcurrentHashMap<String,IndexSchema>(); } zkClientTimeout = Integer.parseInt(System.getProperty("zkClientTimeout", Integer.toString(zkClientTimeout))); - zkSys.initZooKeeper(this, solrHome, zkHost, zkClientTimeout, hostPort, hostContext, host, leaderVoteWait, distribUpdateConnTimeout, distribUpdateSoTimeout); + zkSys.initZooKeeper(this, solrHome, zkHost, zkClientTimeout, hostPort, hostContext, host, leaderVoteWait, genericCoreNodeNames, distribUpdateConnTimeout, distribUpdateSoTimeout); if (isZooKeeperAware() && coreLoadThreads <= 1) { throw new SolrException(ErrorCode.SERVER_ERROR, @@ -1144,6 +1146,8 @@ public class CoreContainer zkSys.getHostContext()); addAttrib(coresAttribs, ConfigSolr.CfgProp.SOLR_LEADERVOTEWAIT, "leaderVoteWait", zkSys.getLeaderVoteWait(), LEADER_VOTE_WAIT); + addAttrib(coresAttribs, ConfigSolr.CfgProp.SOLR_GENERICCORENODENAMES, "genericCoreNodeNames", + Boolean.toString(zkSys.getGenericCoreNodeNames()), "false"); if (transientCacheSize != Integer.MAX_VALUE) { // This test // is a consequence of testing. I really hate it. addAttrib(coresAttribs, ConfigSolr.CfgProp.SOLR_TRANSIENTCACHESIZE, "transientCacheSize", @@ -1221,18 +1225,7 @@ public class CoreContainer } public void preRegisterInZk(final CoreDescriptor p) { - try { - zkSys.getZkController().preRegister(p); - } catch (KeeperException e) { - log.error("", e); - throw new ZooKeeperException( - SolrException.ErrorCode.SERVER_ERROR, "", e); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - log.error("", e); - throw new ZooKeeperException( - SolrException.ErrorCode.SERVER_ERROR, "", e); - } + zkSys.getZkController().preRegister(p); } public String getSolrHome() { Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java?rev=1498702&r1=1498701&r2=1498702&view=diff ============================================================================== --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java (original) +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java Mon Jul 1 21:31:19 2013 @@ -167,7 +167,6 @@ public abstract class DirectoryFactory i / public abstract void release(Directory directory) throws IOException; - /** * Normalize a given path. * @@ -229,5 +228,21 @@ public abstract class DirectoryFactory i } return isSuccess; } - + + /** + * If your implementation can count on delete-on-last-close semantics + * or throws an exception when trying to remove a file in use, return + * false (eg NFS). Otherwise, return true. Defaults to returning false. + * + * @return true if factory impl requires that Searcher's explicitly + * reserve commit points. + */ + public boolean searchersReserveCommitPoints() { + return false; + } + + public String getDataHome(CoreDescriptor cd) throws IOException { + // by default, we go off the instance directory + return normalize(SolrResourceLoader.normalizeDir(cd.getInstanceDir()) + cd.getDataDir()); + } } Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/EphemeralDirectoryFactory.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/EphemeralDirectoryFactory.java?rev=1498702&r1=1498701&r2=1498702&view=diff ============================================================================== --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/EphemeralDirectoryFactory.java (original) +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/EphemeralDirectoryFactory.java Mon Jul 1 21:31:19 2013 @@ -51,4 +51,16 @@ public abstract class EphemeralDirectory public boolean isAbsolute(String path) { return true; } + + + @Override + public void remove(Directory dir) throws IOException { + // ram dir does not persist its dir anywhere + } + + @Override + public void remove(String path) throws IOException { + // ram dir does not persist its dir anywhere + } + } Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/SolrCore.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/SolrCore.java?rev=1498702&r1=1498701&r2=1498702&view=diff ============================================================================== --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/SolrCore.java (original) +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/SolrCore.java Mon Jul 1 21:31:19 2013 @@ -673,11 +673,10 @@ public final class SolrCore implements S if (dataDir == null) { if (cd.usingDefaultDataDir()) dataDir = config.getDataDir(); if (dataDir == null) { - dataDir = cd.getDataDir(); try { + dataDir = cd.getDataDir(); if (!directoryFactory.isAbsolute(dataDir)) { - dataDir = directoryFactory.normalize(SolrResourceLoader - .normalizeDir(cd.getInstanceDir()) + dataDir); + dataDir = directoryFactory.getDataHome(cd); } } catch (IOException e) { throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, null, e); @@ -748,7 +747,6 @@ public final class SolrCore implements S this.codec = initCodec(solrConfig, schema); if (updateHandler == null) { - initDirectoryFactory(); solrCoreState = new DefaultSolrCoreState(getDirectoryFactory()); } else { solrCoreState = updateHandler.getSolrCoreState(); Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/ZkContainer.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/ZkContainer.java?rev=1498702&r1=1498701&r2=1498702&view=diff ============================================================================== --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/ZkContainer.java (original) +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/ZkContainer.java Mon Jul 1 21:31:19 2013 @@ -55,6 +55,7 @@ public class ZkContainer { private String hostContext; private String host; private String leaderVoteWait; + private Boolean genericCoreNodeNames; private int distribUpdateConnTimeout; public SolrZkServer getZkServer() { @@ -80,6 +81,10 @@ public class ZkContainer { public String getLeaderVoteWait() { return leaderVoteWait; } + + public boolean getGenericCoreNodeNames() { + return genericCoreNodeNames; + } public int getDistribUpdateConnTimeout() { return distribUpdateConnTimeout; @@ -95,7 +100,7 @@ public class ZkContainer { } - public void initZooKeeper(final CoreContainer cc, String solrHome, String zkHost, int zkClientTimeout, String hostPort, String hostContext, String host, String leaderVoteWait, int distribUpdateConnTimeout, int distribUpdateSoTimeout) { + public void initZooKeeper(final CoreContainer cc, String solrHome, String zkHost, int zkClientTimeout, String hostPort, String hostContext, String host, String leaderVoteWait, boolean genericCoreNodeNames, int distribUpdateConnTimeout, int distribUpdateSoTimeout) { ZkController zkController = null; // if zkHost sys property is not set, we are not using ZooKeeper @@ -113,6 +118,7 @@ public class ZkContainer { this.hostContext = hostContext; this.host = host; this.leaderVoteWait = leaderVoteWait; + this.genericCoreNodeNames = genericCoreNodeNames; this.distribUpdateConnTimeout = distribUpdateConnTimeout; this.distribUpdateSoTimeout = distribUpdateSoTimeout; @@ -175,7 +181,7 @@ public class ZkContainer { } zkController = new ZkController(cc, zookeeperHost, zkClientTimeout, zkClientConnectTimeout, host, hostPort, hostContext, - leaderVoteWait, distribUpdateConnTimeout, distribUpdateSoTimeout, + leaderVoteWait, genericCoreNodeNames, distribUpdateConnTimeout, distribUpdateSoTimeout, new CurrentCoreDescriptorProvider() { @Override Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java?rev=1498702&r1=1498701&r2=1498702&view=diff ============================================================================== --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java (original) +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java Mon Jul 1 21:31:19 2013 @@ -362,20 +362,18 @@ public class HttpShardHandler extends Sh Map<String, Replica> sliceShards = slice.getReplicasMap(); // For now, recreate the \| delimited list of equivalent servers - Set<String> liveNodes = clusterState.getLiveNodes(); StringBuilder sliceShardsStr = new StringBuilder(); boolean first = true; - for (ZkNodeProps nodeProps : sliceShards.values()) { - ZkCoreNodeProps coreNodeProps = new ZkCoreNodeProps(nodeProps); - if (!liveNodes.contains(coreNodeProps.getNodeName()) - \|\| !coreNodeProps.getState().equals( + for (Replica replica : sliceShards.values()) { + if (!clusterState.liveNodesContain(replica.getNodeName()) + \|\| !replica.getStr(ZkStateReader.STATE_PROP).equals( ZkStateReader.ACTIVE)) continue; if (first) { first = false; } else { sliceShardsStr.append('\|'); } - String url = coreNodeProps.getCoreUrl(); + String url = ZkCoreNodeProps.getCoreUrl(replica); if (url.startsWith("http://")) url = url.substring(7); sliceShardsStr.append(url); Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java?rev=1498702&r1=1498701&r2=1498702&view=diff ============================================================================== --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java (original) +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java Mon Jul 1 21:31:19 2013 @@ -192,7 +192,13 @@ public class SolrIndexSearcher extends I this.name = "Searcher@" + Integer.toHexString(hashCode()) + (name!=null ? " "+name : ""); log.info("Opening " + this.name); - Directory dir = this.reader.directory(); + if (directoryFactory.searchersReserveCommitPoints()) { + // reserve commit point for life of searcher + core.getDeletionPolicy().saveCommitPoint( + reader.getIndexCommit().getGeneration()); + } + + Directory dir = getIndexReader().directory(); this.reserveDirectory = reserveDirectory; this.createdDirectory = r == null; @@ -330,12 +336,18 @@ public class SolrIndexSearcher extends I // super.close(); // can't use super.close() since it just calls reader.close() and that may only be called once // per reader (even if incRef() was previously called). + + long cpg = reader.getIndexCommit().getGeneration(); try { if (closeReader) reader.decRef(); } catch (Throwable t) { SolrException.log(log, "Problem dec ref'ing reader", t); } + if (directoryFactory.searchersReserveCommitPoints()) { + core.getDeletionPolicy().releaseCommitPoint(cpg); + } + for (SolrCache cache : cacheList) { cache.close(); } Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/ReusedBufferedIndexOutput.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/ReusedBufferedIndexOutput.java?rev=1498702&r1=1497072&r2=1498702&view=diff ============================================================================== --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/ReusedBufferedIndexOutput.java (original) +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/ReusedBufferedIndexOutput.java Mon Jul 1 21:31:19 2013 @@ -92,6 +92,23 @@ public abstract class ReusedBufferedInde protected abstract void seekInternal(long pos) throws IOException; @Override + public void seek(long pos) throws IOException { + if (pos > fileLength) { + fileLength = pos; + } + + if (pos >= bufferStart && pos < (bufferStart + bufferLength)) + bufferPosition = (int)(pos - bufferStart); // seek within buffer + else { + flushBufferToCache(); + bufferStart = pos; + bufferPosition = 0; + bufferLength = 0; + seekInternal(pos); + } + } + + @Override public long length() throws IOException { return fileLength; } Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/hdfs/HdfsDirectory.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/hdfs/HdfsDirectory.java?rev=1498702&r1=1497072&r2=1498702&view=diff ============================================================================== --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/hdfs/HdfsDirectory.java (original) +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/hdfs/HdfsDirectory.java Mon Jul 1 21:31:19 2013 @@ -244,6 +244,11 @@ public class HdfsDirectory extends Direc } @Override + public void seek(long pos) throws IOException { + writer.seek(pos); + } + + @Override public void writeByte(byte b) throws IOException { writer.writeByte(b); } Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/hdfs/NullIndexOutput.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/hdfs/NullIndexOutput.java?rev=1498702&r1=1497072&r2=1498702&view=diff ============================================================================== --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/hdfs/NullIndexOutput.java (original) +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/hdfs/NullIndexOutput.java Mon Jul 1 21:31:19 2013 @@ -46,6 +46,12 @@ public class NullIndexOutput extends Ind return length; } + + @Override + public void seek(long pos) throws IOException { + this.pos = pos; + } + @Override public void writeByte(byte b) throws IOException { pos++;
Mime	Unnamed text/plain (inline, 7-Bit, 84967 bytes)
	View raw message