Return-Path: X-Original-To: apmail-lucene-commits-archive@www.apache.org Delivered-To: apmail-lucene-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 5D3861727D for ; Wed, 2 Sep 2015 13:06:39 +0000 (UTC) Received: (qmail 49579 invoked by uid 500); 2 Sep 2015 13:06:35 -0000 Mailing-List: contact commits-help@lucene.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@lucene.apache.org Delivered-To: mailing list commits@lucene.apache.org Received: (qmail 49150 invoked by uid 99); 2 Sep 2015 13:06:35 -0000 Received: from eris.apache.org (HELO hades.apache.org) (140.211.11.105) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 02 Sep 2015 13:06:35 +0000 Received: from hades.apache.org (localhost [127.0.0.1]) by hades.apache.org (ASF Mail Server at hades.apache.org) with ESMTP id 01F07AC1BF5 for ; Wed, 2 Sep 2015 13:06:35 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit Subject: svn commit: r1700800 [9/24] - in /lucene/dev/branches/lucene6699: ./ dev-tools/ dev-tools/eclipse/ dev-tools/idea/.idea/ dev-tools/scripts/ lucene/ lucene/analysis/ lucene/analysis/common/ lucene/analysis/common/src/java/org/apache/lucene/analysis/ar/ ... Date: Wed, 02 Sep 2015 13:06:22 -0000 To: commits@lucene.apache.org From: mikemccand@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20150902130635.01F07AC1BF5@hades.apache.org> Modified: lucene/dev/branches/lucene6699/solr/contrib/langid/src/test-files/langid/solr/collection1/conf/solrconfig-languageidentifier.xml URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/contrib/langid/src/test-files/langid/solr/collection1/conf/solrconfig-languageidentifier.xml?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/contrib/langid/src/test-files/langid/solr/collection1/conf/solrconfig-languageidentifier.xml (original) +++ lucene/dev/branches/lucene6699/solr/contrib/langid/src/test-files/langid/solr/collection1/conf/solrconfig-languageidentifier.xml Wed Sep 2 13:06:13 2015 @@ -62,36 +62,36 @@ - - - - true - name,subject - true - language_s - language_sm - th:thai - 0.5 - fallback - - + + + + true + name,subject + true + language_s + language_sm + th:thai + 0.5 + fallback + + - - - - true - name,subject - true - language_s - language_sm - th:thai - 0.5 - fallback - - + + + + true + name,subject + true + language_s + language_sm + th:thai + 0.5 + fallback + + Modified: lucene/dev/branches/lucene6699/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/GoLive.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/GoLive.java?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/GoLive.java (original) +++ lucene/dev/branches/lucene6699/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/GoLive.java Wed Sep 2 13:06:13 2015 @@ -16,16 +16,6 @@ */ package org.apache.solr.hadoop; -import org.apache.hadoop.fs.FileStatus; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.impl.CloudSolrClient; -import org.apache.solr.client.solrj.impl.HttpSolrClient; -import org.apache.solr.client.solrj.request.CoreAdminRequest; -import org.apache.solr.common.util.ExecutorUtil; -import org.apache.solr.hadoop.MapReduceIndexerTool.Options; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.IOException; import java.util.Arrays; import java.util.HashSet; @@ -35,12 +25,21 @@ import java.util.concurrent.Callable; import java.util.concurrent.CompletionService; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorCompletionService; -import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import org.apache.hadoop.fs.FileStatus; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.request.CoreAdminRequest; +import org.apache.solr.common.util.ExecutorUtil; +import org.apache.solr.hadoop.MapReduceIndexerTool.Options; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * The optional (parallel) GoLive phase merges the output shards of the previous * phase into a set of live customer facing Solr servers, typically a SolrCloud. @@ -164,7 +163,7 @@ class GoLive { success = true; return true; } finally { - shutdownNowAndAwaitTermination(executor); + ExecutorUtil.shutdownAndAwaitTermination(executor); float secs = (System.nanoTime() - start) / (float)(10^9); LOG.info("Live merging of index shards into Solr cluster took " + secs + " secs"); if (success) { @@ -176,25 +175,6 @@ class GoLive { // if an output dir does not exist, we should fail and do no merge? } - - private void shutdownNowAndAwaitTermination(ExecutorService pool) { - pool.shutdown(); // Disable new tasks from being submitted - pool.shutdownNow(); // Cancel currently executing tasks - boolean shutdown = false; - while (!shutdown) { - try { - // Wait a while for existing tasks to terminate - shutdown = pool.awaitTermination(5, TimeUnit.SECONDS); - } catch (InterruptedException ie) { - // Preserve interrupt status - Thread.currentThread().interrupt(); - } - if (!shutdown) { - pool.shutdownNow(); // Cancel currently executing tasks - } - } - } - private static final class Request { Exception e; Modified: lucene/dev/branches/lucene6699/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/HeartBeater.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/HeartBeater.java?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/HeartBeater.java (original) +++ lucene/dev/branches/lucene6699/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/HeartBeater.java Wed Sep 2 13:06:13 2015 @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. Modified: lucene/dev/branches/lucene6699/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/SolrMapper.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/SolrMapper.java?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/SolrMapper.java (original) +++ lucene/dev/branches/lucene6699/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/SolrMapper.java Wed Sep 2 13:06:13 2015 @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. Modified: lucene/dev/branches/lucene6699/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/SolrOutputFormat.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/SolrOutputFormat.java?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/SolrOutputFormat.java (original) +++ lucene/dev/branches/lucene6699/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/SolrOutputFormat.java Wed Sep 2 13:06:13 2015 @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. Modified: lucene/dev/branches/lucene6699/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/SolrRecordWriter.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/SolrRecordWriter.java?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/SolrRecordWriter.java (original) +++ lucene/dev/branches/lucene6699/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/SolrRecordWriter.java Wed Sep 2 13:06:13 2015 @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. Modified: lucene/dev/branches/lucene6699/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/SolrReducer.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/SolrReducer.java?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/SolrReducer.java (original) +++ lucene/dev/branches/lucene6699/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/SolrReducer.java Wed Sep 2 13:06:13 2015 @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. Modified: lucene/dev/branches/lucene6699/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/Utils.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/Utils.java?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/Utils.java (original) +++ lucene/dev/branches/lucene6699/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/Utils.java Wed Sep 2 13:06:13 2015 @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. Modified: lucene/dev/branches/lucene6699/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/ZooKeeperInspector.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/ZooKeeperInspector.java?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/ZooKeeperInspector.java (original) +++ lucene/dev/branches/lucene6699/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/ZooKeeperInspector.java Wed Sep 2 13:06:13 2015 @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. Modified: lucene/dev/branches/lucene6699/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MorphlineGoLiveMiniMRTest.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MorphlineGoLiveMiniMRTest.java?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MorphlineGoLiveMiniMRTest.java (original) +++ lucene/dev/branches/lucene6699/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MorphlineGoLiveMiniMRTest.java Wed Sep 2 13:06:13 2015 @@ -128,8 +128,8 @@ public class MorphlineGoLiveMiniMRTest e this.inputAvroFile2 = "sample-statuses-20120906-141433.avro"; this.inputAvroFile3 = "sample-statuses-20120906-141433-medium.avro"; - sliceCount = TEST_NIGHTLY ? 7 : 3; - fixShardCount(TEST_NIGHTLY ? 7 : 3); + sliceCount = TEST_NIGHTLY ? 5 : 3; + fixShardCount(TEST_NIGHTLY ? 5 : 3); } @BeforeClass @@ -458,7 +458,7 @@ public class MorphlineGoLiveMiniMRTest e args = new String[]{ "--output-dir=" + outDir.toString(), "--mappers=3", - "--reducers=12", + "--reducers=6", "--fanout=2", "--verbose", "--go-live", @@ -520,7 +520,7 @@ public class MorphlineGoLiveMiniMRTest e // try using zookeeper with replication String replicatedCollection = "replicated_collection"; if (TEST_NIGHTLY) { - createCollection(replicatedCollection, 7, 3, 9); + createCollection(replicatedCollection, 3, 3, 3); } else { createCollection(replicatedCollection, 2, 3, 2); } @@ -536,7 +536,7 @@ public class MorphlineGoLiveMiniMRTest e "--solr-home-dir=" + MINIMR_CONF_DIR.getAbsolutePath(), "--output-dir=" + outDir.toString(), "--mappers=3", - "--reducers=22", + "--reducers=12", "--fanout=2", "--verbose", "--go-live", @@ -651,7 +651,7 @@ public class MorphlineGoLiveMiniMRTest e } if (TEST_NIGHTLY) { - createCollection(replicatedCollection, 7, 3, 9); + createCollection(replicatedCollection, 3, 3, 3); } else { createCollection(replicatedCollection, 2, 3, 2); } Modified: lucene/dev/branches/lucene6699/solr/contrib/morphlines-cell/src/java/org/apache/solr/morphlines/cell/SolrCellBuilder.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/contrib/morphlines-cell/src/java/org/apache/solr/morphlines/cell/SolrCellBuilder.java?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/contrib/morphlines-cell/src/java/org/apache/solr/morphlines/cell/SolrCellBuilder.java (original) +++ lucene/dev/branches/lucene6699/solr/contrib/morphlines-cell/src/java/org/apache/solr/morphlines/cell/SolrCellBuilder.java Wed Sep 2 13:06:13 2015 @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. Modified: lucene/dev/branches/lucene6699/solr/contrib/morphlines-core/src/java/org/apache/solr/morphlines/solr/TokenizeTextBuilder.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/contrib/morphlines-core/src/java/org/apache/solr/morphlines/solr/TokenizeTextBuilder.java?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/contrib/morphlines-core/src/java/org/apache/solr/morphlines/solr/TokenizeTextBuilder.java (original) +++ lucene/dev/branches/lucene6699/solr/contrib/morphlines-core/src/java/org/apache/solr/morphlines/solr/TokenizeTextBuilder.java Wed Sep 2 13:06:13 2015 @@ -117,24 +117,6 @@ public final class TokenizeTextBuilder i } - - // Copied from org.apache.lucene.document.Field.java from lucene-4.3.0 - /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ private static final class ReusableStringReader extends Reader { private int pos = 0, size = 0; private String s = null; Modified: lucene/dev/branches/lucene6699/solr/contrib/morphlines-core/src/test-files/solr/collection1/conf/schema.xml URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/contrib/morphlines-core/src/test-files/solr/collection1/conf/schema.xml?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/contrib/morphlines-core/src/test-files/solr/collection1/conf/schema.xml (original) +++ lucene/dev/branches/lucene6699/solr/contrib/morphlines-core/src/test-files/solr/collection1/conf/schema.xml Wed Sep 2 13:06:13 2015 @@ -184,8 +184,8 @@ + removes stop words from case-insensitive "stopwords.txt" + (empty by default), and down cases. At query time only, it + also applies synonyms. --> @@ -331,11 +331,11 @@ /> - + - + --> @@ -347,23 +347,23 @@ /> - + - + --> @@ -415,7 +415,7 @@ + each token, to enable more efficient leading wildcard queries. --> @@ -492,10 +492,10 @@ a token of "foo|1.4" would be indexed as "foo" with a payload of 1.4f Attributes of the DelimitedPayloadTokenFilterFactory : "delimiter" - a one character delimiter. Default is | (pipe) - "encoder" - how to encode the following value into a playload - float -> org.apache.lucene.analysis.payloads.FloatEncoder, - integer -> o.a.l.a.p.IntegerEncoder - identity -> o.a.l.a.p.IdentityEncoder + "encoder" - how to encode the following value into a playload + float -> org.apache.lucene.analysis.payloads.FloatEncoder, + integer -> o.a.l.a.p.IntegerEncoder + identity -> o.a.l.a.p.IdentityEncoder Fully Qualified class name implementing PayloadEncoder, Encoder must have a no arg constructor. --> @@ -516,10 +516,10 @@ --> - + - + - + - + Modified: lucene/dev/branches/lucene6699/solr/contrib/morphlines-core/src/test-files/solr/collection1/conf/solrconfig.xml URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/contrib/morphlines-core/src/test-files/solr/collection1/conf/solrconfig.xml?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/contrib/morphlines-core/src/test-files/solr/collection1/conf/solrconfig.xml (original) +++ lucene/dev/branches/lucene6699/solr/contrib/morphlines-core/src/test-files/solr/collection1/conf/solrconfig.xml Wed Sep 2 13:06:13 2015 @@ -955,7 +955,7 @@ 0.01 Modified: lucene/dev/branches/lucene6699/solr/contrib/morphlines-core/src/test-files/solr/minimr/conf/schema.xml URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/contrib/morphlines-core/src/test-files/solr/minimr/conf/schema.xml?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/contrib/morphlines-core/src/test-files/solr/minimr/conf/schema.xml (original) +++ lucene/dev/branches/lucene6699/solr/contrib/morphlines-core/src/test-files/solr/minimr/conf/schema.xml Wed Sep 2 13:06:13 2015 @@ -196,8 +196,8 @@ + removes stop words from case-insensitive "stopwords.txt" + (empty by default), and down cases. At query time only, it + also applies synonyms. --> @@ -346,11 +346,11 @@ /> - + - + --> @@ -362,23 +362,23 @@ /> - + - + --> @@ -430,7 +430,7 @@ + each token, to enable more efficient leading wildcard queries. --> @@ -505,10 +505,10 @@ a token of "foo|1.4" would be indexed as "foo" with a payload of 1.4f Attributes of the DelimitedPayloadTokenFilterFactory : "delimiter" - a one character delimiter. Default is | (pipe) - "encoder" - how to encode the following value into a playload - float -> org.apache.lucene.analysis.payloads.FloatEncoder, - integer -> o.a.l.a.p.IntegerEncoder - identity -> o.a.l.a.p.IdentityEncoder + "encoder" - how to encode the following value into a playload + float -> org.apache.lucene.analysis.payloads.FloatEncoder, + integer -> o.a.l.a.p.IntegerEncoder + identity -> o.a.l.a.p.IdentityEncoder Fully Qualified class name implementing PayloadEncoder, Encoder must have a no arg constructor. --> @@ -529,10 +529,10 @@ --> - + - + - + - + Modified: lucene/dev/branches/lucene6699/solr/contrib/morphlines-core/src/test-files/solr/minimr/conf/solrconfig.xml URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/contrib/morphlines-core/src/test-files/solr/minimr/conf/solrconfig.xml?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/contrib/morphlines-core/src/test-files/solr/minimr/conf/solrconfig.xml (original) +++ lucene/dev/branches/lucene6699/solr/contrib/morphlines-core/src/test-files/solr/minimr/conf/solrconfig.xml Wed Sep 2 13:06:13 2015 @@ -974,7 +974,7 @@ 0.01 Modified: lucene/dev/branches/lucene6699/solr/contrib/morphlines-core/src/test-files/solr/mrunit/conf/schema.xml URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/contrib/morphlines-core/src/test-files/solr/mrunit/conf/schema.xml?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/contrib/morphlines-core/src/test-files/solr/mrunit/conf/schema.xml (original) +++ lucene/dev/branches/lucene6699/solr/contrib/morphlines-core/src/test-files/solr/mrunit/conf/schema.xml Wed Sep 2 13:06:13 2015 @@ -196,8 +196,8 @@ + removes stop words from case-insensitive "stopwords.txt" + (empty by default), and down cases. At query time only, it + also applies synonyms. --> @@ -346,11 +346,11 @@ /> - + - + --> @@ -362,23 +362,23 @@ /> - + - + --> @@ -430,7 +430,7 @@ + each token, to enable more efficient leading wildcard queries. --> @@ -505,10 +505,10 @@ a token of "foo|1.4" would be indexed as "foo" with a payload of 1.4f Attributes of the DelimitedPayloadTokenFilterFactory : "delimiter" - a one character delimiter. Default is | (pipe) - "encoder" - how to encode the following value into a playload - float -> org.apache.lucene.analysis.payloads.FloatEncoder, - integer -> o.a.l.a.p.IntegerEncoder - identity -> o.a.l.a.p.IdentityEncoder + "encoder" - how to encode the following value into a playload + float -> org.apache.lucene.analysis.payloads.FloatEncoder, + integer -> o.a.l.a.p.IntegerEncoder + identity -> o.a.l.a.p.IdentityEncoder Fully Qualified class name implementing PayloadEncoder, Encoder must have a no arg constructor. --> @@ -529,10 +529,10 @@ --> - + - + - + - + Modified: lucene/dev/branches/lucene6699/solr/contrib/morphlines-core/src/test-files/solr/mrunit/conf/solrconfig.xml URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/contrib/morphlines-core/src/test-files/solr/mrunit/conf/solrconfig.xml?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/contrib/morphlines-core/src/test-files/solr/mrunit/conf/solrconfig.xml (original) +++ lucene/dev/branches/lucene6699/solr/contrib/morphlines-core/src/test-files/solr/mrunit/conf/solrconfig.xml Wed Sep 2 13:06:13 2015 @@ -978,7 +978,7 @@ 0.01 Modified: lucene/dev/branches/lucene6699/solr/contrib/morphlines-core/src/test-files/solr/solrcelltest/collection1/conf/schema.xml URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/contrib/morphlines-core/src/test-files/solr/solrcelltest/collection1/conf/schema.xml?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/contrib/morphlines-core/src/test-files/solr/solrcelltest/collection1/conf/schema.xml (original) +++ lucene/dev/branches/lucene6699/solr/contrib/morphlines-core/src/test-files/solr/solrcelltest/collection1/conf/schema.xml Wed Sep 2 13:06:13 2015 @@ -151,8 +151,8 @@ + removes stop words from case-insensitive "stopwords.txt" + (empty by default), and down cases. At query time only, it + also applies synonyms. --> @@ -298,11 +298,11 @@ /> - + - + --> @@ -314,23 +314,23 @@ /> - + - + --> @@ -382,7 +382,7 @@ + each token, to enable more efficient leading wildcard queries. --> @@ -459,10 +459,10 @@ a token of "foo|1.4" would be indexed as "foo" with a payload of 1.4f Attributes of the DelimitedPayloadTokenFilterFactory : "delimiter" - a one character delimiter. Default is | (pipe) - "encoder" - how to encode the following value into a playload - float -> org.apache.lucene.analysis.payloads.FloatEncoder, - integer -> o.a.l.a.p.IntegerEncoder - identity -> o.a.l.a.p.IdentityEncoder + "encoder" - how to encode the following value into a playload + float -> org.apache.lucene.analysis.payloads.FloatEncoder, + integer -> o.a.l.a.p.IntegerEncoder + identity -> o.a.l.a.p.IdentityEncoder Fully Qualified class name implementing PayloadEncoder, Encoder must have a no arg constructor. --> @@ -483,10 +483,10 @@ --> - + - + - + - + Modified: lucene/dev/branches/lucene6699/solr/contrib/morphlines-core/src/test-files/solr/solrcelltest/collection1/conf/solrconfig.xml URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/contrib/morphlines-core/src/test-files/solr/solrcelltest/collection1/conf/solrconfig.xml?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/contrib/morphlines-core/src/test-files/solr/solrcelltest/collection1/conf/solrconfig.xml (original) +++ lucene/dev/branches/lucene6699/solr/contrib/morphlines-core/src/test-files/solr/solrcelltest/collection1/conf/solrconfig.xml Wed Sep 2 13:06:13 2015 @@ -955,7 +955,7 @@ 0.01 Modified: lucene/dev/branches/lucene6699/solr/contrib/morphlines-core/src/test-files/solr/solrcloud/conf/solrconfig.xml URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/contrib/morphlines-core/src/test-files/solr/solrcloud/conf/solrconfig.xml?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/contrib/morphlines-core/src/test-files/solr/solrcloud/conf/solrconfig.xml (original) +++ lucene/dev/branches/lucene6699/solr/contrib/morphlines-core/src/test-files/solr/solrcloud/conf/solrconfig.xml Wed Sep 2 13:06:13 2015 @@ -977,7 +977,7 @@ 0.01 Modified: lucene/dev/branches/lucene6699/solr/contrib/morphlines-core/src/test-files/test-documents/boilerplate.html URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/contrib/morphlines-core/src/test-files/test-documents/boilerplate.html?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/contrib/morphlines-core/src/test-files/test-documents/boilerplate.html (original) +++ lucene/dev/branches/lucene6699/solr/contrib/morphlines-core/src/test-files/test-documents/boilerplate.html Wed Sep 2 13:06:13 2015 @@ -20,22 +20,22 @@ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> - - Title + + Title - - - + + +
- - - - - -
boilerplatetext
-
+ + + + + +
boilerplatetext
+

This is the real meat of the page, Modified: lucene/dev/branches/lucene6699/solr/contrib/morphlines-core/src/test-files/test-documents/testXML.xml URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/contrib/morphlines-core/src/test-files/test-documents/testXML.xml?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/contrib/morphlines-core/src/test-files/test-documents/testXML.xml (original) +++ lucene/dev/branches/lucene6699/solr/contrib/morphlines-core/src/test-files/test-documents/testXML.xml Wed Sep 2 13:06:13 2015 @@ -17,32 +17,32 @@ --> - Tika test document + Tika test document - Rida Benjelloun + Rida Benjelloun - Java + Java - XML + XML - XSLT + XSLT - JDOM + JDOM - Indexation + Indexation - Framework d'indexation des documents XML, HTML, PDF etc.. + Framework d'indexation des documents XML, HTML, PDF etc.. - http://www.apache.org + http://www.apache.org - 2000-12-01T00:00:00.000Z + 2000-12-01T00:00:00.000Z - test + test - application/msword + application/msword - Fr + Fr - Archimède et Lius à Châteauneuf testing chars en été + Archimède et Lius à Châteauneuf testing chars en été \ No newline at end of file Modified: lucene/dev/branches/lucene6699/solr/contrib/uima/src/resources/org/apache/uima/desc/HmmTagger.xml URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/contrib/uima/src/resources/org/apache/uima/desc/HmmTagger.xml?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/contrib/uima/src/resources/org/apache/uima/desc/HmmTagger.xml (original) +++ lucene/dev/branches/lucene6699/solr/contrib/uima/src/resources/org/apache/uima/desc/HmmTagger.xml Wed Sep 2 13:06:13 2015 @@ -1,21 +1,21 @@ org.apache.uima.java @@ -24,9 +24,9 @@ Hidden Markov Model - Part of Speech Tagger A configuration of the HmmTaggerAnnotator that looks for - parts of speech of identified tokens within existing - Sentence and Token annotations. See also - WhitespaceTokenizer.xml. + parts of speech of identified tokens within existing + Sentence and Token annotations. See also + WhitespaceTokenizer.xml. 1.0 The Apache Software Foundation @@ -55,7 +55,7 @@ posTag contains part-of-speech of a - corresponding token + corresponding token uima.cas.String Modified: lucene/dev/branches/lucene6699/solr/contrib/uima/src/resources/org/apache/uima/desc/TextCategorizationAEDescriptor.xml URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/contrib/uima/src/resources/org/apache/uima/desc/TextCategorizationAEDescriptor.xml?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/contrib/uima/src/resources/org/apache/uima/desc/TextCategorizationAEDescriptor.xml (original) +++ lucene/dev/branches/lucene6699/solr/contrib/uima/src/resources/org/apache/uima/desc/TextCategorizationAEDescriptor.xml Wed Sep 2 13:06:13 2015 @@ -1,22 +1,22 @@ Modified: lucene/dev/branches/lucene6699/solr/contrib/uima/src/resources/org/apache/uima/desc/WhitespaceTokenizer.xml URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/contrib/uima/src/resources/org/apache/uima/desc/WhitespaceTokenizer.xml?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/contrib/uima/src/resources/org/apache/uima/desc/WhitespaceTokenizer.xml (original) +++ lucene/dev/branches/lucene6699/solr/contrib/uima/src/resources/org/apache/uima/desc/WhitespaceTokenizer.xml Wed Sep 2 13:06:13 2015 @@ -1,115 +1,115 @@ - - org.apache.uima.java - - true - - org.apache.uima.annotator.WhitespaceTokenizer - - - - WhitespaceTokenizer - - creates token and sentence annotations for whitespace - separated languages - - 1.0 - The Apache Software Foundation - - - - SofaNames - - The Sofa names the annotator should work on. If no - names are specified, the annotator works on the - default sofa. - - String - true - false - - - - - - - - - - - org.apache.uima.TokenAnnotation - Single token annotation - uima.tcas.Annotation - - - tokenType - token type - uima.cas.String - - - - - - org.apache.uima.SentenceAnnotation - sentence annotation - uima.tcas.Annotation - - - - - - - - - - - - - org.apache.uima.TokenAnnotation - - org.apache.uima.TokenAnnotation:tokentype - - org.apache.uima.SentenceAnnotation - - - x-unspecified - - - + xmlns="http://uima.apache.org/resourceSpecifier"> + + org.apache.uima.java + + true + + org.apache.uima.annotator.WhitespaceTokenizer + + + + WhitespaceTokenizer + + creates token and sentence annotations for whitespace + separated languages + + 1.0 + The Apache Software Foundation + + + + SofaNames + + The Sofa names the annotator should work on. If no + names are specified, the annotator works on the + default sofa. + + String + true + false + + + + + + + + + + + org.apache.uima.TokenAnnotation + Single token annotation + uima.tcas.Annotation + + + tokenType + token type + uima.cas.String + + + + + + org.apache.uima.SentenceAnnotation + sentence annotation + uima.tcas.Annotation + + + + + + + + + + + + + org.apache.uima.TokenAnnotation + + org.apache.uima.TokenAnnotation:tokentype + + org.apache.uima.SentenceAnnotation + + + x-unspecified + + + - + Modified: lucene/dev/branches/lucene6699/solr/contrib/velocity/src/java/org/apache/solr/response/PageTool.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/contrib/velocity/src/java/org/apache/solr/response/PageTool.java?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/contrib/velocity/src/java/org/apache/solr/response/PageTool.java (original) +++ lucene/dev/branches/lucene6699/solr/contrib/velocity/src/java/org/apache/solr/response/PageTool.java Wed Sep 2 13:06:13 2015 @@ -18,7 +18,6 @@ package org.apache.solr.response; import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.search.DocList; import org.apache.solr.search.DocSlice; import org.apache.solr.common.SolrDocumentList; @@ -50,7 +49,7 @@ public class PageTool { results_found = doc_slice.matches(); start = doc_slice.offset(); } else if(docs instanceof ResultContext) { - DocList dl = ((ResultContext) docs).docs; + DocList dl = ((ResultContext) docs).getDocList(); results_found = dl.matches(); start = dl.offset(); } else if(docs instanceof SolrDocumentList) { Modified: lucene/dev/branches/lucene6699/solr/contrib/velocity/src/java/org/apache/solr/response/VelocityResponseWriter.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/contrib/velocity/src/java/org/apache/solr/response/VelocityResponseWriter.java?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/contrib/velocity/src/java/org/apache/solr/response/VelocityResponseWriter.java (original) +++ lucene/dev/branches/lucene6699/solr/contrib/velocity/src/java/org/apache/solr/response/VelocityResponseWriter.java Wed Sep 2 13:06:13 2015 @@ -25,7 +25,9 @@ import java.io.StringWriter; import java.io.Writer; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.HashMap; import java.util.Locale; +import java.util.Map; import java.util.Properties; import java.util.ResourceBundle; @@ -45,7 +47,6 @@ import org.apache.velocity.runtime.Runti import org.apache.velocity.runtime.resource.loader.ClasspathResourceLoader; import org.apache.velocity.tools.ConversionUtils; import org.apache.velocity.tools.generic.ComparisonDateTool; -import org.apache.velocity.tools.generic.ContextTool; import org.apache.velocity.tools.generic.DisplayTool; import org.apache.velocity.tools.generic.EscapeTool; import org.apache.velocity.tools.generic.ListTool; @@ -84,6 +85,7 @@ public class VelocityResponseWriter impl private static final Logger log = LoggerFactory.getLogger(VelocityResponseWriter.class); private static final SolrVelocityLogger velocityLogger = new SolrVelocityLogger(log); private Properties velocityInitProps = new Properties(); + private Map customTools = new HashMap(); @Override public void init(NamedList args) { @@ -112,6 +114,14 @@ public class VelocityResponseWriter impl solrResourceLoaderEnabled = (null == srle ? true : srle); initPropertiesFileName = (String) args.get(PROPERTIES_FILE); + + NamedList tools = (NamedList)args.get("tools"); + if (tools != null) { + for(Object t : tools) { + Map.Entry tool = (Map.Entry)t; + customTools.put(tool.getKey().toString(), tool.getValue().toString()); + } + } } @Override @@ -183,10 +193,8 @@ public class VelocityResponseWriter impl private VelocityContext createContext(SolrQueryRequest request, SolrQueryResponse response) { VelocityContext context = new VelocityContext(); - context.put("request", request); - // Register useful Velocity "tools" - context.put("log", log); // TODO: add test + context.put("log", log); // TODO: add test; TODO: should this be overridable with a custom "log" named tool? context.put("esc", new EscapeTool()); context.put("date", new ComparisonDateTool()); context.put("list", new ListTool()); @@ -198,6 +206,25 @@ public class VelocityResponseWriter impl request.getCore().getSolrConfig().getResourceLoader().getClassLoader(), request.getParams().get(LOCALE))); +/* + // Custom tools, specified in config as: + + + com.example.solr.velocity.MyTool + + + + +*/ + // Custom tools can override any of the built-in tools provided above, by registering one with the same name + for(String name : customTools.keySet()) { + context.put(name, SolrCore.createInstance(customTools.get(name), Object.class, "VrW custom tool", request.getCore(), request.getCore().getResourceLoader())); + } + + // custom tools _cannot_ override context objects added below, like $request and $response + // TODO: at least log a warning when one of the *fixed* tools classes in name with a custom one, currently silently ignored + + // Turn the SolrQueryResponse into a SolrResponse. // QueryResponse has lots of conveniences suitable for a view // Problem is, which SolrResponse class to use? @@ -221,6 +248,8 @@ public class VelocityResponseWriter impl rsp = new SolrResponseBase(); rsp.setResponse(parsedResponse); } + + context.put("request", request); context.put("response", rsp); return context; @@ -284,6 +313,8 @@ public class VelocityResponseWriter impl engine.setProperty(RuntimeConstants.RESOURCE_LOADER, StringUtils.join(loaders,',')); + engine.setProperty(RuntimeConstants.INPUT_ENCODING, "UTF-8"); + // bring in any custom properties too engine.init(velocityInitProps); Modified: lucene/dev/branches/lucene6699/solr/contrib/velocity/src/resources/_macros.vm URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/contrib/velocity/src/resources/_macros.vm?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/contrib/velocity/src/resources/_macros.vm (original) +++ lucene/dev/branches/lucene6699/solr/contrib/velocity/src/resources/_macros.vm Wed Sep 2 13:06:13 2015 @@ -40,7 +40,7 @@ #set($pad = " ... ") #end #else - #foreach($v in $doc.getFieldValues($f))$esc.html($v)#end + $esc.html($display.list($doc.getFieldValues($f), ", ")) #end #end Modified: lucene/dev/branches/lucene6699/solr/contrib/velocity/src/test-files/velocity/solr/collection1/conf/solrconfig.xml URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/contrib/velocity/src/test-files/velocity/solr/collection1/conf/solrconfig.xml?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/contrib/velocity/src/test-files/velocity/solr/collection1/conf/solrconfig.xml (original) +++ lucene/dev/branches/lucene6699/solr/contrib/velocity/src/test-files/velocity/solr/collection1/conf/solrconfig.xml Wed Sep 2 13:06:13 2015 @@ -35,4 +35,20 @@ velocity-init.properties + + + + true + + + + org.apache.solr.velocity.MockTool + + + org.apache.solr.velocity.MockTool + + + org.apache.solr.velocity.MockTool + + Modified: lucene/dev/branches/lucene6699/solr/contrib/velocity/src/test/org/apache/solr/velocity/VelocityResponseWriterTest.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/contrib/velocity/src/test/org/apache/solr/velocity/VelocityResponseWriterTest.java?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/contrib/velocity/src/test/org/apache/solr/velocity/VelocityResponseWriterTest.java (original) +++ lucene/dev/branches/lucene6699/solr/contrib/velocity/src/test/org/apache/solr/velocity/VelocityResponseWriterTest.java Wed Sep 2 13:06:13 2015 @@ -93,6 +93,11 @@ public class VelocityResponseWriterTest } @Test + public void testEncoding() throws Exception { + assertEquals("éñçø∂îñg", h.query(req("q","*:*", "wt","velocity",VelocityResponseWriter.TEMPLATE,"encoding"))); + } + + @Test public void testMacros() throws Exception { // tests that a macro in a custom macros.vm is visible assertEquals("test_macro_SUCCESS", h.query(req("q","*:*", "wt","velocity",VelocityResponseWriter.TEMPLATE,"test_macro_visible"))); @@ -115,6 +120,29 @@ public class VelocityResponseWriterTest } @Test + public void testCustomTools() throws Exception { + assertEquals("", h.query(req("q","*:*", "wt","velocity",VelocityResponseWriter.TEMPLATE,"custom_tool"))); + assertEquals("** LATERALUS **", h.query(req("q","*:*", "wt","velocityWithCustomTools",VelocityResponseWriter.TEMPLATE,"t", + SolrParamResourceLoader.TEMPLATE_PARAM_PREFIX+"t", "$mytool.star(\"LATERALUS\")"))); + + // Does $log get overridden? + assertEquals("** log overridden **", h.query(req("q","*:*", "wt","velocityWithCustomTools",VelocityResponseWriter.TEMPLATE,"t", + SolrParamResourceLoader.TEMPLATE_PARAM_PREFIX+"t", "$log.star(\"log overridden\")"))); + + // Does $response get overridden? actual blank response because of the bang on $! reference that silences bogus $-references + assertEquals("", h.query(req("q","*:*", "wt","velocityWithCustomTools",VelocityResponseWriter.TEMPLATE,"t", + SolrParamResourceLoader.TEMPLATE_PARAM_PREFIX+"t", "$!response.star(\"response overridden??\")"))); + + // Custom tools can also have a SolrCore-arg constructor because they are instantiated with SolrCore.createInstance + // TODO: do we really need to support this? no great loss, as a custom tool could take a SolrCore object as a parameter to + // TODO: any method, so one could do $mytool.my_method($request.core) + // I'm currently inclined to make this feature undocumented/unsupported, as we may want to instantiate classes + // in a different manner that only supports no-arg constructors, commented (passing) test case out +// assertEquals("collection1", h.query(req("q","*:*", "wt","velocityWithCustomTools",VelocityResponseWriter.TEMPLATE,"t", +// SolrParamResourceLoader.TEMPLATE_PARAM_PREFIX+"t", "$mytool.core.name"))); + } + + @Test public void testLocaleFeature() throws Exception { assertEquals("Color", h.query(req("q", "*:*", "wt", "velocity", VelocityResponseWriter.TEMPLATE, "locale", VelocityResponseWriter.LOCALE,"en_US"))); Modified: lucene/dev/branches/lucene6699/solr/core/src/java/org/apache/solr/client/solrj/embedded/EmbeddedSolrServer.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/core/src/java/org/apache/solr/client/solrj/embedded/EmbeddedSolrServer.java?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/core/src/java/org/apache/solr/client/solrj/embedded/EmbeddedSolrServer.java (original) +++ lucene/dev/branches/lucene6699/solr/core/src/java/org/apache/solr/client/solrj/embedded/EmbeddedSolrServer.java Wed Sep 2 13:06:13 2015 @@ -189,9 +189,9 @@ public class EmbeddedSolrServer extends public void writeResults(ResultContext ctx, JavaBinCodec codec) throws IOException { // write an empty list... SolrDocumentList docs = new SolrDocumentList(); - docs.setNumFound(ctx.docs.matches()); - docs.setStart(ctx.docs.offset()); - docs.setMaxScore(ctx.docs.maxScore()); + docs.setNumFound(ctx.getDocList().matches()); + docs.setStart(ctx.getDocList().offset()); + docs.setMaxScore(ctx.getDocList().maxScore()); codec.writeSolrDocumentList(docs); // This will transform