lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From h..@apache.org
Subject svn commit: r1503797 [9/19] - in /lucene/dev/branches/lucene3069: ./ dev-tools/ dev-tools/idea/.idea/libraries/ dev-tools/idea/lucene/suggest/ dev-tools/idea/solr/contrib/dataimporthandler/ dev-tools/idea/solr/core/src/test/ dev-tools/maven/ dev-tools/...
Date Tue, 16 Jul 2013 17:00:18 GMT
Modified: lucene/dev/branches/lucene3069/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/CHANGES.txt?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/CHANGES.txt (original)
+++ lucene/dev/branches/lucene3069/solr/CHANGES.txt Tue Jul 16 17:00:05 2013
@@ -24,7 +24,7 @@ $Id$
 
 Versions of Major Components
 ---------------------
-Apache Tika 1.3
+Apache Tika 1.4
 Carrot2 3.6.2
 Velocity 1.7 and Velocity Tools 2.0
 Apache UIMA 2.3.1
@@ -33,7 +33,8 @@ Apache ZooKeeper 3.4.5
 Upgrading from Solr 4.x
 ----------------------
 
-TBD...
+The "file" attribute of infoStream in solrconfig.xml is removed. Control this 
+via your logging configuration (org.apache.solr.update.LoggingInfoStream) instead.
 
 Detailed Change List
 ----------------------
@@ -46,11 +47,34 @@ Other Changes
 
 * SOLR-4792: Stop shipping a .war. (Robert Muir)
 
+
+==================  4.5.0 ==================
+
+Versions of Major Components
+---------------------
+Apache Tika 1.4
+Carrot2 3.6.2
+Velocity 1.7 and Velocity Tools 2.0
+Apache UIMA 2.3.1
+Apache ZooKeeper 3.4.5
+
+Upgrading from Solr 4.4.0
+----------------------
+
+Detailed Change List
+----------------------
+
+Other Changes
+----------------------
+
+* SOLR-4914: Factor out core list persistence and discovery into a
+  new CoresLocator interface. (Alan Woodward)
+
 ==================  4.4.0 ==================
 
 Versions of Major Components
 ---------------------
-Apache Tika 1.3
+Apache Tika 1.4
 Carrot2 3.6.2
 Velocity 1.7 and Velocity Tools 2.0
 Apache UIMA 2.3.1
@@ -59,9 +83,30 @@ Apache ZooKeeper 3.4.5
 Upgrading from Solr 4.3.0
 ----------------------
 
+* TieredMergePolicy and the various subtypes of LogMergePolicy no longer have 
+  an explicit "setUseCompoundFile" method.  Instead the behavior of new 
+  segments is determined by the IndexWriter configuration, and the MergePolicy 
+  is only consulted to determine if merge segements should use the compound 
+  file format (based on the value of "setNoCFSRatio").  If you have explicitly 
+  configured one of these classes using <mergePolicy> and include an init arg 
+  like this...
+     <bool name="useCompoundFile">true</bool>
+  ...this will now be treated as if you specified...
+     <useCompoundFile>true</useCompoundFile> 
+  ...directly on the <indexConfig> (overriding any value already set using that
+  syntax) and a warning will be logged to updated your configuration.  Users 
+  with an explicitly declared <mergePolicy> are encouraged to review the 
+  current javadocs for their MergePolicy subclass and review their configured 
+  options carefully.  See SOLR-4941, SOLR-4934 and LUCENE-5038 for more 
+  information.
+
 * SOLR-4778: The signature of LogWatcher.registerListener has changed, from
   (ListenerConfig, CoreContainer) to (ListenerConfig).  Users implementing their
   own LogWatcher classes will need to change their code accordingly.
+
+* LUCENE-5063: ByteField and ShortField have been deprecated and will be removed
+  in 5.0. If you are still using these field types, you should migrate your
+  fields to TrieIntField.
   
 Detailed Change List
 ----------------------
@@ -71,8 +116,9 @@ New Features
 
 * SOLR-3251: Dynamically add fields to schema. (Steve Rowe, Robert Muir, yonik)   
 
-* SOLR-4761: Add option to plugin a merged segment warmer into solrconfig.xml
-  (Mark Miller, Mike McCandless, Robert Muir)
+* SOLR-4761, SOLR-4976: Add option to plugin a merged segment warmer into solrconfig.xml. 
+  Info about segments warmed in the background is available via infostream.
+  (Mark Miller, Ryan Ernst, Mike McCandless, Robert Muir)
 
 * SOLR-3240: Add "spellcheck.collateMaxCollectDocs" option so that when testing
   potential Collations against the index, SpellCheckComponent will only collect
@@ -95,6 +141,52 @@ New Features
   <bool name="fieldNameMatchesSchemaField">false</bool> to an update
   processor's configuration in solrconfig.xml.  (Steve Rowe, hossman)
 
+* SOLR-4921: Admin UI now supports adding documents to Solr (gsingers, steffkes)
+
+* SOLR-4916: Add support to write and read Solr index files and transaction log
+  files to and from HDFS. (phunt, Mark Miller, Greg Chanan)
+  
+* SOLR-4892: Add FieldMutatingUpdateProcessorFactory subclasses 
+  Parse{Date,Integer,Long,Float,Double,Boolean}UpdateProcessorFactory. These
+  factories have a default selector that matches all fields that either don’t
+  match any schema field, or are in the schema with the corresponding
+  typeClass. If they see a value that is not a CharSequence, or can't parse
+  the value, they leave it as is. For multi-valued fields, these processors
+  will not convert any values unless all are first successfully parsed, or 
+  already are instances of the target class. Ordering the processors, e.g.
+  [Boolean, Long, Double, Date] will allow e.g. values ["2", "5", "8.6"] to
+  be left alone by the Boolean and Long processors, but then converted by the
+  Double processor.  (Steve Rowe, hossman)
+
+* SOLR-4972: Add PUT command to ZkCli tool. (Roman Shaposhnik via Mark Miller)
+
+* SOLR-4973: Adding getter method for defaultCollection on CloudSolrServer.
+  (Furkan KAMACI via Mark Miller)
+  
+* SOLR-4897: Add solr/example/example-schemaless/, an example config set
+  for schemaless mode. (Steve Rowe)
+
+* SOLR-4655: Add option to have Overseer assign generic node names so that
+  new addresses can host shards without naming confusion. (Mark Miller, Anshum Gupta)
+
+* SOLR-4977: Add option to send IndexWriter's infostream to the logging system.
+  (Ryan Ernst via Robert Muir)
+
+* SOLR-4693: A "deleteshard" collections API that unloads all replicas of a given
+  shard and then removes it from the cluster state. It will remove only those shards
+  which are INACTIVE or have no range (created for custom sharding).
+  (Anshum Gupta, shalin)
+
+* SOLR-5003: CSV Update Handler supports optionally adding the line number/row id to 
+  a document (gsingers)
+
+* SOLR-5010: Add support for creating copy fields to the Schema REST API (gsingers)
+
+* SOLR-4991: Register QParserPlugins as SolrInfoMBeans (ehatcher)
+
+* SOLR-4943: Add a new system wide info admin handler that exposes the system info
+  that could previously only be retrieved using a SolrCore. (Mark Miller)
+
 Bug Fixes
 ----------------------
 
@@ -141,6 +233,109 @@ Bug Fixes
 
 * SOLR-4925 : Collection create throws NPE when 'numShards' param is missing (Noble Paul)
 
+* SOLR-4910: persisting solr.xml is broken. More stringent testing of persistence fixed
+  up a number of issues and several bugs with persistence. Among them are
+  > don't persisting implicit properties
+  > should persist zkHost in the <solr> tag (user's list)
+  > reloading a core that has transient="true" returned an error. reload should load
+    a transient core if it's not yet loaded.
+  > No longer persisting loadOnStartup or transient core properties if they were not
+    specified in the original solr.xml
+  > Testing flushed out the fact that you couldn't swap a core marked transient=true
+    loadOnStartup=false because it hadn't been loaded yet.
+  > SOLR-4862, CREATE fails to persist schema, config, and dataDir
+  > SOLR-4363, not persisting coreLoadThreads in <solr> tag
+  > SOLR-3900, logWatcher properties not persisted
+  > SOLR-4850, cores defined as loadOnStartup=true, transient=false can't be searched
+  (Erick Erickson)
+
+* SOLR-4923: Commits to non leaders as part of a request that also contain updates
+  can execute out of order. (hossman, Ricardo Merizalde, Mark Miller)
+
+* SOLR-4932: persisting solr.xml saves some parameters it shouldn't when they weren't
+  defined in the original. Benign since the default values are saved, but still incorrect.
+  (Erick Erickson, thanks Shawn Heisey for helping test!)
+
+* SOLR-4934, SOLR-4941: Fix handling of <mergePolicy> init arg 
+  "useCompoundFile" needed after changes in LUCENE-5038 (hossman)
+
+* SOLR-4456: Admin UI: Displays dashboard even if Solr is down (steffkes)
+
+* SOLR-4949: UI Analysis page dropping characters from input box (steffkes)
+
+* SOLR-4960: Fix race conditions in shutdown of CoreContainer
+  and getCore that could cause a request to attempt to use a core that
+  has shut down. (yonik)
+
+* SOLR-4926: Fixed rare replication bug that normally only manifested when 
+  using compound file format. (yonik, Mark Miller)
+  
+* SOLR-4974: Outgrowth of SOLR-4960 that includes transient cores and pending cores
+  (Erick Erickson)
+
+* SOLR-3369: shards.tolerant=true is broken for group queries
+  (Russell Black, Martijn van Groningen, Jabouille jean Charles, Ryan McKinley via shalin)
+
+* SOLR-4452: Hunspell stemmer should not merge duplicate dictionary entries (janhoy)
+
+* SOLR-5000: ManagedIndexSchema doesn't persist uniqueKey tag after calling addFields
+  method. (Jun Ohtani, Steve Rowe)
+  
+* SOLR-4982: Creating a core while referencing system properties looks like it loses files
+  Actually, instanceDir, config, dataDir and schema are not dereferenced properly
+  when creating cores that reference sys vars (e.g. &dataDir=${dir}). In the dataDir
+  case in particular this leads to the index being put in a directory literally named
+  ${dir} but on restart the sysvar will be properly dereferenced.
+
+* SOLR-4788: Multiple Entities DIH delta import: dataimporter.[entityName].last_index_time
+  is empty. (chakming wong, James Dyer via shalin)
+
+* SOLR-4978: Time is stripped from datetime column when imported into Solr date field
+  if convertType=true. (Bill Au, shalin)
+
+* SOLR-5019: spurious ConcurrentModificationException when spell check component
+  was in use with filters. (yonik)
+  
+* SOLR-5018: The Overseer should avoid publishing the state for collections that do not 
+  exist under the /collections zk node. (Mark Miller)
+
+* SOLR-5028,SOLR-5029: ShardHandlerFactory was not being created properly when
+  using new-style solr.xml, and was not being persisted properly when using
+  old-style.  (Tomás Fernández Löbbe, Ryan Ernst, Alan Woodward)
+
+* SOLR-4997: The splitshard api doesn't call commit on new sub shards before
+  switching shard states. Multiple bugs related to sub shard recovery and
+  replication are also fixed. (shalin)
+
+* SOLR-5034: A facet.query that parses or analyzes down to a null Query would
+  throw a NPE. Fixed. (David Smiley)
+  
+* SOLR-5039: Admin/Schema Browser displays -1 for term counts for multiValued fields.
+
+Optimizations
+----------------------
+
+* SOLR-4923: Commit to all nodes in a collection in parallel rather than locally and
+  then to all other nodes. (hossman, Ricardo Merizalde, Mark Miller)
+
+* SOLR-3838: Admin UI - Multiple filter queries are not supported in Query UI (steffkes)
+
+* SOLR-4719 : Admin UI - Default to wt=json on Query-Screen (steffkes)
+
+* SOLR-4611: Admin UI - Analysis-Urls with empty parameters create empty result table
+  (steffkes)
+
+* SOLR-4955: Admin UI - Show address bar on top for Schema + Config (steffkes)
+
+* SOLR-4412: New parameter langid.lcmap to map detected language code to be placed 
+  in "language" field (janhoy)
+
+* SOLR-4815: Admin-UI - DIH: Let "commit" be checked by default (steffkes)
+
+* SOLR-5002: optimize numDocs(Query,DocSet) when filterCache is null (Robert Muir)
+
+* SOLR-5012: optimize search with filter when filterCache is null (Robert Muir)
+
 Other Changes
 ----------------------
 
@@ -171,7 +366,24 @@ Other Changes
   (Philip Hoy via Robert Muir)
   
 * SOLR-4224: Refactor JavaBinCodec input stream definition to enhance reuse.
-  (phunt via Mark Miller)  
+  (phunt via Mark Miller)
+
+* SOLR-4931: SolrDeletionPolicy onInit and onCommit methods changed to override
+  exact signatures (with generics) from IndexDeletionPolicy (shalin)
+
+* SOLR-4942: test improvements to randomize use of compound files (hosman)
+
+* SOLR-4966: CSS, JS and other files in webapp without license (uschindler,
+  steffkes)
+
+* SOLR-4986: Upgrade to Tika 1.4 (Markus Jelsma via janhoy)
+
+* SOLR-4948, SOLR-5009: Tidied up CoreContainer construction logic.
+  (Alan Woodward, Uwe Schindler, Steve Rowe)
+
+* LUCENE-5107: Properties files by Solr are now written in UTF-8 encoding,
+  Unicode is no longer escaped. Reading of legacy properties files with
+  \u escapes is still possible.  (Uwe Schindler, Robert Muir)
 
 ==================  4.3.1 ==================
 
@@ -258,7 +470,7 @@ Bug Fixes
 
 * SOLR-4744: Update failure on sub shard is not propagated to clients by parent
   shard (Anshum Gupta, yonik, shalin)
-
+  
 Other Changes
 ----------------------
 

Modified: lucene/dev/branches/lucene3069/solr/NOTICE.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/NOTICE.txt?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/NOTICE.txt (original)
+++ lucene/dev/branches/lucene3069/solr/NOTICE.txt Tue Jul 16 17:00:05 2013
@@ -11,6 +11,8 @@ including, but not limited to:
   - Apache Lucene Java
   - Apache Commons
   - Apache Geronimo (stax API)
+  - Apache Blur
+  - Apache Hadoop
 
 This product includes the JQuery JavaScript library created by John Resig.
 Copyright (c) 2010 John Resig, http://jquery.com/
@@ -21,6 +23,42 @@ Copyright (c) 2012, Michael Bostock, htt
 This product includes the highlight.js Javascript library created by Ivan Sagalaev
 Copyright (c) 2006, Ivan Sagalaev, https://github.com/isagalaev/highlight.js
 
+This product includes the ZeroClipboard.js Javascript library created by Jon Rohan, James M. Greene
+Copyright (c) 2012 Jon Rohan, James M. Greene, https://github.com/zeroclipboard/ZeroClipboard
+
+This product includes the Chosen Javascript library created by Patrick Filler
+Copyright (c) 2011-2013 by Harvest, https://github.com/harvesthq/chosen
+
+This product includes jquery.ajaxfileupload.js Javascript library created by Jordan Feldstein
+Copyright (c) 2011 Jordan Feldstein, https://github.com/jfeldstein/jQuery.AjaxFileUpload.js
+
+This product includes jquery.blockUI.js Javascript library created by Mike Alsup
+Copyright (c) 2007-2013 M. Alsup https://github.com/malsup/blockui/
+
+This product includes jquery.cookie.js Javascript library created by Klaus Hartl
+Copyright (c) 2013 Klaus Hartl, https://github.com/carhartl/jquery-cookie
+
+This product includes jquery.form Javascript library created by Mike Alsup
+Copyright 2006-2013 (c) M. Alsup, https://github.com/malsup/form/
+
+This product includes the jstree Javascript library created by Ivan Bozhanov
+Copyright (c) 2013 Ivan Bozhanov, https://github.com/vakata/jstree
+
+This product includes the Sammy.js Javascript library created by Aaron Quint
+Copyright (c) 2008 Aaron Quint, Quirkey NYC, LLC, https://github.com/quirkey/sammy
+
+This product includes jquery.timeago.js Javascript library by Ryan McGeary
+Copyright (c) 2008-2013, Ryan McGeary, https://github.com/rmm5t/jquery-timeago
+
+This product includes linker.js Javascript library created by Michalis Tzikas & Vasilis Lolos
+Copyright (C) 2011 by Michalis Tzikas & Vasilis Lolos, https://github.com/lolos/jquery-Linker/
+
+This product includes require.js Javascript library created by James Burke
+Copyright (C) 2010-2013 James Burke, https://github.com/jrburke/requirejs
+
+This product includes fugue icons created by Yusuke Kamiyamane
+Copyright (C) 2013 Yusuke Kamiyamane, https://github.com/yusukekamiyamane/fugue-icons
+
 stax-utils library: https://stax-utils.dev.java.net/
 Copyright (c) 2004, Christian Niles, unit12.net
 Copyright (c) 2004, Sun Microsystems, Inc.
@@ -36,6 +74,12 @@ License: Common Development and Distribu
 HSQL Database (HSQLDB): http://hsqldb.org/
 License: http://hsqldb.org/web/hsqlLicense.html
 
+Jersey Core: https://jersey.java.net/
+License: Common Development and Distribution License (CDDL) v1.0 (https://glassfish.dev.java.net/public/CDDLv1.0.html)
+
+The HdfsDirectory and BlockDirectory were derived from
+the Apache Blur incubating project and are Apache License 2.0.
+
 =========================================================================
 ==  Apache Lucene Notice                                               ==
 =========================================================================
@@ -510,3 +554,11 @@ Restlet is a registered trademark of Res
 This product contains software developed by the Restlet project.
 
 See http://www.restlet.org/
+
+=========================================================================
+==     Protocol Buffers Notice                                         ==
+=========================================================================
+
+Protocol Buffers - Google's data interchange format
+Copyright 2008 Google Inc.
+http://code.google.com/apis/protocolbuffers/

Modified: lucene/dev/branches/lucene3069/solr/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/build.xml?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/build.xml (original)
+++ lucene/dev/branches/lucene3069/solr/build.xml Tue Jul 16 17:00:05 2013
@@ -55,7 +55,7 @@
     </delete>
     <echo>See ${example}/README.txt for how to run the Solr example configuration.</echo>
   </target>
-
+  
   <target name="run-example" depends="example"
           description="Run Solr interactively, via Jetty.  -Dexample.debug=true to enable JVM debugger">
     <property name="example.solr.home" location="example/solr"/>
@@ -251,7 +251,7 @@
   <!-- TODO: Make the forbidden API checks per module! Also remove failOnMissingClasses="false" below!!! -->
   <target name="-forbidden-apis-classpath">
     <path id="forbidden-apis.classpath">
-      <fileset dir="${basedir}" includes="**/lib/*.jar"/>
+      <fileset dir="${basedir}" includes="**/lib/*.jar,**/test-lib/*.jar"/>
       <dirset dir="${basedir}/build" includes="**/classes/*"/>
       <path refid="solr.lucene.libs"/>
       <path refid="classpath"/>
@@ -265,8 +265,7 @@
       <bundledSignatures name="jdk-deprecated-${javac.target}"/>
       <bundledSignatures name="commons-io-unsafe-${commons-io.version}"/>
       <signaturesFileSet dir="${common.dir}/tools/forbiddenApis">
-        <include name="executors.txt" />
-        <include name="chars.txt" />
+        <include name="base.txt" />
         <include name="servlet-api.txt" />
       </signaturesFileSet>
       <fileset dir="${basedir}/build">
@@ -292,12 +291,13 @@
 
   <!-- rat sources -->
   <!-- rat-sources-typedef is *not* a useless dependency. do not remove -->
-  <target name="rat-sources" depends="rat-sources-typedef">
+  <target name="rat-sources" depends="rat-sources-typedef,common.rat-sources">
     <subant target="rat-sources" inheritall="false" >
       <propertyset refid="uptodate.and.compiled.properties"/>
       <fileset dir="core" includes="build.xml"/>
       <fileset dir="solrj" includes="build.xml"/>
       <fileset dir="test-framework" includes="build.xml"/>
+      <fileset dir="webapp" includes="build.xml"/>
     </subant>
     <contrib-crawl target="rat-sources" failonerror="true"/>
   </target>

Modified: lucene/dev/branches/lucene3069/solr/common-build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/common-build.xml?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/common-build.xml (original)
+++ lucene/dev/branches/lucene3069/solr/common-build.xml Tue Jul 16 17:00:05 2013
@@ -96,6 +96,12 @@
 
   <path id="solr.test.base.classpath">
     <pathelement path="${common-solr.dir}/build/solr-test-framework/classes/java"/>
+    <fileset dir="${common-solr.dir}/test-framework/lib">
+      <include name="*.jar"/>
+      <exclude name="junit-*.jar" />
+      <exclude name="randomizedtesting-runner-*.jar" />
+      <exclude name="ant*.jar" />
+    </fileset>
   	<pathelement path="${build.dir}/test-files"/>
   	<path refid="test.base.classpath"/>
   </path>
@@ -117,6 +123,12 @@
     </sequential>
   </macrodef>
 
+  <!-- we don't want to run HDFS tests on Windows, because they require Cygwin.
+    If you have Cygwin, you can override this property on command line: -->
+  <condition property="tests.disableHdfs" value="true">
+    <os family="windows"/>
+  </condition>
+
   <target name="validate" depends="compile-tools">
   </target>
 

Modified: lucene/dev/branches/lucene3069/solr/contrib/analysis-extras/ivy.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/contrib/analysis-extras/ivy.xml?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/contrib/analysis-extras/ivy.xml (original)
+++ lucene/dev/branches/lucene3069/solr/contrib/analysis-extras/ivy.xml Tue Jul 16 17:00:05 2013
@@ -20,9 +20,9 @@
     <info organisation="org.apache.solr" module="analysis-extras"/>
     <dependencies>
       <dependency org="com.ibm.icu" name="icu4j" rev="49.1" transitive="false"/>
-      <dependency org="org.carrot2" name="morfologik-polish" rev="1.5.5" transitive="false"/>
-      <dependency org="org.carrot2" name="morfologik-fsa" rev="1.5.5" transitive="false"/>
-      <dependency org="org.carrot2" name="morfologik-stemming" rev="1.5.5" transitive="false"/>
+      <dependency org="org.carrot2" name="morfologik-polish" rev="1.6.0" transitive="false"/>
+      <dependency org="org.carrot2" name="morfologik-fsa" rev="1.6.0" transitive="false"/>
+      <dependency org="org.carrot2" name="morfologik-stemming" rev="1.6.0" transitive="false"/>
       <exclude org="*" ext="*" matcher="regexp" type="${ivy.exclude.types}"/> 
     </dependencies>
 </ivy-module>

Modified: lucene/dev/branches/lucene3069/solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java (original)
+++ lucene/dev/branches/lucene3069/solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java Tue Jul 16 17:00:05 2013
@@ -19,7 +19,6 @@ package org.apache.solr.schema;
 
 import java.io.IOException;
 import java.io.InputStream;
-import java.io.StringReader;
 import java.util.Map;
 
 import org.apache.commons.io.IOUtils;
@@ -27,8 +26,6 @@ import org.apache.lucene.analysis.Analyz
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
 import org.apache.lucene.collation.ICUCollationKeyAnalyzer;
-import org.apache.lucene.index.GeneralField;
-import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.StorableField;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.SortField;
@@ -240,7 +237,7 @@ public class ICUCollationField extends F
     TokenStream source;
       
     try {
-      source = analyzer.tokenStream(field, new StringReader(part));
+      source = analyzer.tokenStream(field, part);
       source.reset();
     } catch (IOException e) {
       throw new RuntimeException("Unable to initialize TokenStream to analyze range part: " + part, e);

Modified: lucene/dev/branches/lucene3069/solr/contrib/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/solrconfig-icucollate.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/contrib/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/solrconfig-icucollate.xml?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/contrib/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/solrconfig-icucollate.xml (original)
+++ lucene/dev/branches/lucene3069/solr/contrib/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/solrconfig-icucollate.xml Tue Jul 16 17:00:05 2013
@@ -19,6 +19,9 @@
 
 <config>
   <luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
+  <indexConfig>
+    <useCompoundFile>${useCompoundFile:false}</useCompoundFile>
+  </indexConfig>
   <requestHandler name="standard" class="solr.StandardRequestHandler"></requestHandler>
   <directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.RAMDirectoryFactory}"/>
 </config>

Modified: lucene/dev/branches/lucene3069/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/solrconfig.xml?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/solrconfig.xml (original)
+++ lucene/dev/branches/lucene3069/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/solrconfig.xml Tue Jul 16 17:00:05 2013
@@ -28,6 +28,7 @@
 
   <indexConfig>
     <lockType>single</lockType>
+    <useCompoundFile>${useCompoundFile:false}</useCompoundFile>
   </indexConfig>
   
   <!--	Enables JMX if and only if an existing MBeanServer is found, use 

Modified: lucene/dev/branches/lucene3069/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/solr/collection1/conf/dataimport-solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/solr/collection1/conf/dataimport-solrconfig.xml?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/solr/collection1/conf/dataimport-solrconfig.xml (original)
+++ lucene/dev/branches/lucene3069/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/solr/collection1/conf/dataimport-solrconfig.xml Tue Jul 16 17:00:05 2013
@@ -18,6 +18,9 @@
 
 <config>
   <luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
+  <indexConfig>
+    <useCompoundFile>${useCompoundFile:false}</useCompoundFile>
+  </indexConfig>
 
   <!-- Used to specify an alternate directory to hold all index data
        other than the default ./data under the Solr home.
@@ -26,8 +29,6 @@
 
   <directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.RAMDirectoryFactory}"/>
 
-  <!-- <indexConfig> section could go here, but we want the defaults -->
-
   <!-- the default high-performance update handler -->
   <updateHandler class="solr.DirectUpdateHandler2">
 

Modified: lucene/dev/branches/lucene3069/solr/contrib/dataimporthandler/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/contrib/dataimporthandler/build.xml?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/contrib/dataimporthandler/build.xml (original)
+++ lucene/dev/branches/lucene3069/solr/contrib/dataimporthandler/build.xml Tue Jul 16 17:00:05 2013
@@ -17,7 +17,7 @@
     limitations under the License.
  -->
 
-<project name="solr-dataimporthandler" default="default">
+<project name="solr-dataimporthandler" default="default" xmlns:ivy="antlib:org.apache.ivy.ant">
 	
   <description>
     Data Import Handler
@@ -26,6 +26,20 @@
   <!-- the tests have some parallel problems: writability to single copy of dataimport.properties -->
   <property name="tests.jvms.override" value="1"/>
 
+  <property name="test.lib.dir" location="test-lib"/>
+  
   <import file="../contrib-build.xml"/>
 
+  <path id="test.classpath">
+    <path refid="solr.test.base.classpath"/>
+    <fileset dir="${test.lib.dir}" includes="*.jar"/>
+  </path>
+
+  <target name="resolve" depends="ivy-availability-check,ivy-fail,ivy-configure">
+    <sequential>
+      <ivy:retrieve conf="compile" type="jar,bundle" sync="${ivy.sync}" log="download-only"/>
+      <ivy:retrieve conf="test" type="jar,bundle,test" sync="${ivy.sync}" log="download-only"
+                    pattern="${test.lib.dir}/[artifact]-[revision](-[classifier]).[ext]"/>
+    </sequential>
+  </target>
 </project>

Modified: lucene/dev/branches/lucene3069/solr/contrib/dataimporthandler/ivy.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/contrib/dataimporthandler/ivy.xml?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/contrib/dataimporthandler/ivy.xml (original)
+++ lucene/dev/branches/lucene3069/solr/contrib/dataimporthandler/ivy.xml Tue Jul 16 17:00:05 2013
@@ -17,10 +17,17 @@
    under the License.    
 -->
 <ivy-module version="2.0">
-    <info organisation="org.apache.solr" module="dataimporthandler"/>
-    <dependencies>
-      <dependency org="javax.activation" name="activation" rev="1.1" transitive="false"/>
-      <dependency org="javax.mail" name="mail" rev="1.4.1" transitive="false"/>
-      <exclude org="*" ext="*" matcher="regexp" type="${ivy.exclude.types}"/> 
-    </dependencies>
+  <info organisation="org.apache.solr" module="dataimporthandler"/>
+  <configurations>
+    <conf name="compile" transitive="false"/>
+    <conf name="test" transitive="false"/>
+  </configurations>
+  <dependencies>
+    <dependency org="javax.activation" name="activation" rev="1.1" conf="compile->*"/>
+    <dependency org="javax.mail" name="mail" rev="1.4.1"  conf="compile->*"/>
+    
+    <dependency org="org.easymock" name="easymock" rev="3.0" conf="test->*"/>
+
+    <exclude org="*" ext="*" matcher="regexp" type="${ivy.exclude.types}"/> 
+  </dependencies>
 </ivy-module>

Modified: lucene/dev/branches/lucene3069/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java (original)
+++ lucene/dev/branches/lucene3069/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java Tue Jul 16 17:00:05 2013
@@ -134,13 +134,15 @@ public class DocBuilder {
       indexerNamespace.put(INDEX_START_TIME, dataImporter.getIndexStartTime());
       indexerNamespace.put("request", new HashMap<String,Object>(reqParams.getRawParams()));
       for (Entity entity : dataImporter.getConfig().getEntities()) {
-        String key = entity.getName() + "." + SolrWriter.LAST_INDEX_KEY;
-        Object lastIndex = persistedProperties.get(key);
-        if (lastIndex != null && lastIndex instanceof Date) {
-          indexerNamespace.put(key, lastIndex);
+        Map<String, Object> entityNamespace = new HashMap<String, Object>();        
+        String key = SolrWriter.LAST_INDEX_KEY;
+        Object lastIndex = persistedProperties.get(entity.getName() + "." + key);
+        if (lastIndex != null) {
+          entityNamespace.put(SolrWriter.LAST_INDEX_KEY, lastIndex);
         } else  {
-          indexerNamespace.put(key, EPOCH);
+          entityNamespace.put(SolrWriter.LAST_INDEX_KEY, EPOCH);
         }
+        indexerNamespace.put(entity.getName(), entityNamespace);
       }
       resolver.addNamespace(ConfigNameConstants.IMPORTER_NS_SHORT, indexerNamespace);
       resolver.addNamespace(ConfigNameConstants.IMPORTER_NS, indexerNamespace);

Modified: lucene/dev/branches/lucene3069/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/JdbcDataSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/JdbcDataSource.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/JdbcDataSource.java (original)
+++ lucene/dev/branches/lucene3069/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/JdbcDataSource.java Tue Jul 16 17:00:05 2013
@@ -333,7 +333,7 @@ public class JdbcDataSource extends
               result.put(colName, resultSet.getDouble(colName));
               break;
             case Types.DATE:
-              result.put(colName, resultSet.getDate(colName));
+              result.put(colName, resultSet.getTimestamp(colName));
               break;
             case Types.BOOLEAN:
               result.put(colName, resultSet.getBoolean(colName));

Modified: lucene/dev/branches/lucene3069/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SimplePropertiesWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SimplePropertiesWriter.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SimplePropertiesWriter.java (original)
+++ lucene/dev/branches/lucene3069/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SimplePropertiesWriter.java Tue Jul 16 17:00:05 2013
@@ -22,9 +22,10 @@ import static org.apache.solr.handler.da
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
-import java.io.IOException;
 import java.io.InputStream;
-import java.io.OutputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.Date;
@@ -33,6 +34,7 @@ import java.util.Locale;
 import java.util.Map;
 import java.util.Properties;
 
+import org.apache.lucene.util.IOUtils;
 import org.apache.solr.core.SolrCore;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -179,7 +181,7 @@ public class SimplePropertiesWriter exte
   
   @Override
   public void persist(Map<String, Object> propObjs) {
-    OutputStream propOutput = null;    
+    Writer propOutput = null;    
     Properties existingProps = mapToProperties(readIndexerProperties());    
     Properties newProps = mapToProperties(propObjs);
     try {
@@ -189,18 +191,14 @@ public class SimplePropertiesWriter exte
         filePath += File.separator;
       }
       filePath += filename;
-      propOutput = new FileOutputStream(filePath);
+      propOutput = new OutputStreamWriter(new FileOutputStream(filePath), IOUtils.CHARSET_UTF_8);
       existingProps.store(propOutput, null);
       log.info("Wrote last indexed time to " + filename);
     } catch (Exception e) {
       throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
           "Unable to persist Index Start Time", e);
     } finally {
-      try {
-        if (propOutput != null) propOutput.close();
-      } catch (IOException e) {
-        propOutput = null;
-      }
+      IOUtils.closeWhileHandlingException(propOutput);
     }
   }
   
@@ -215,16 +213,12 @@ public class SimplePropertiesWriter exte
       }
       filePath += filename;
       propInput = new FileInputStream(filePath);
-      props.load(propInput);
+      props.load(new InputStreamReader(propInput, IOUtils.CHARSET_UTF_8));
       log.info("Read " + filename);
     } catch (Exception e) {
       log.warn("Unable to read: " + filename);
     } finally {
-      try {
-        if (propInput != null) propInput.close();
-      } catch (IOException e) {
-        propInput = null;
-      }
+      IOUtils.closeWhileHandlingException(propInput);
     }    
     return propertiesToMap(props);
   }

Modified: lucene/dev/branches/lucene3069/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ZKPropertiesWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ZKPropertiesWriter.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ZKPropertiesWriter.java (original)
+++ lucene/dev/branches/lucene3069/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ZKPropertiesWriter.java Tue Jul 16 17:00:05 2013
@@ -16,11 +16,12 @@
  */
 package org.apache.solr.handler.dataimport;
 
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
+import java.io.StringReader;
+import java.io.StringWriter;
 import java.util.Map;
 import java.util.Properties;
 
+import org.apache.lucene.util.IOUtils;
 import org.apache.solr.common.cloud.SolrZkClient;
 import org.apache.zookeeper.KeeperException.NodeExistsException;
 import org.slf4j.Logger;
@@ -63,10 +64,10 @@ public class ZKPropertiesWriter extends 
   public void persist(Map<String, Object> propObjs) {
     Properties existing = mapToProperties(readIndexerProperties());
     existing.putAll(mapToProperties(propObjs));
-    ByteArrayOutputStream output = new ByteArrayOutputStream();
+    StringWriter output = new StringWriter();
     try {
-      existing.store(output, "");
-      byte[] bytes = output.toByteArray();
+      existing.store(output, null);
+      byte[] bytes = output.toString().getBytes(IOUtils.CHARSET_UTF_8);
       if (!zkClient.exists(path, false)) {
         try {
           zkClient.makePath(path, false);
@@ -89,8 +90,7 @@ public class ZKPropertiesWriter extends 
     try {
       byte[] data = zkClient.getData(path, null, null, false);
       if (data != null) {
-        ByteArrayInputStream input = new ByteArrayInputStream(data);
-        props.load(input);
+        props.load(new StringReader(new String(data, "UTF-8")));
       }
     } catch (Throwable e) {
       log.warn(

Modified: lucene/dev/branches/lucene3069/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/contentstream-solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/contentstream-solrconfig.xml?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/contentstream-solrconfig.xml (original)
+++ lucene/dev/branches/lucene3069/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/contentstream-solrconfig.xml Tue Jul 16 17:00:05 2013
@@ -18,6 +18,9 @@
 
 <config>
   <luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
+  <indexConfig>
+    <useCompoundFile>${useCompoundFile:false}</useCompoundFile>
+  </indexConfig>
 
   <!-- Used to specify an alternate directory to hold all index data
        other than the default ./data under the Solr home.
@@ -26,8 +29,6 @@
 
   <directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.RAMDirectoryFactory}"/>
 
-  <!-- <indexConfig> section could go here, but we want the defaults -->
-
   <!-- the default high-performance update handler -->
   <updateHandler class="solr.DirectUpdateHandler2">
 

Modified: lucene/dev/branches/lucene3069/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-nodatasource-solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-nodatasource-solrconfig.xml?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-nodatasource-solrconfig.xml (original)
+++ lucene/dev/branches/lucene3069/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-nodatasource-solrconfig.xml Tue Jul 16 17:00:05 2013
@@ -28,6 +28,7 @@
 
   <indexConfig>
     <lockType>single</lockType>
+    <useCompoundFile>${useCompoundFile:false}</useCompoundFile>
   </indexConfig>
 
   <!-- the default high-performance update handler -->

Modified: lucene/dev/branches/lucene3069/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-solrconfig.xml?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-solrconfig.xml (original)
+++ lucene/dev/branches/lucene3069/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-solrconfig.xml Tue Jul 16 17:00:05 2013
@@ -18,6 +18,9 @@
 
 <config>
   <luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
+  <indexConfig>
+    <useCompoundFile>${useCompoundFile:false}</useCompoundFile>
+  </indexConfig>
 
   <!-- Used to specify an alternate directory to hold all index data
        other than the default ./data under the Solr home.
@@ -26,8 +29,6 @@
 
   <directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.RAMDirectoryFactory}"/>
 
-  <!-- <indexConfig> section could go here, but we want the defaults -->
-
   <!-- the default high-performance update handler -->
   <updateHandler class="solr.DirectUpdateHandler2">
 

Modified: lucene/dev/branches/lucene3069/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta.java (original)
+++ lucene/dev/branches/lucene3069/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta.java Tue Jul 16 17:00:05 2013
@@ -165,13 +165,13 @@ public class TestSqlEntityProcessorDelta
   @Override
   protected String deltaQueriesPersonTable() {
     return 
-        "deletedPkQuery=''SELECT ID FROM PEOPLE WHERE DELETED='Y' AND last_modified &gt;='${dih.last_index_time}' '' " +
+        "deletedPkQuery=''SELECT ID FROM PEOPLE WHERE DELETED='Y' AND last_modified &gt;='${dih.People.last_index_time}' '' " +
         "deltaImportQuery=''SELECT ID, NAME, COUNTRY_CODE FROM PEOPLE where ID=${dih.delta.ID} '' " +
         "deltaQuery=''" +
-        "SELECT ID FROM PEOPLE WHERE DELETED!='Y' AND last_modified &gt;='${dih.last_index_time}' " +
+        "SELECT ID FROM PEOPLE WHERE DELETED!='Y' AND last_modified &gt;='${dih.People.last_index_time}' " +
         (useParentDeltaQueryParam ? "" : 
         "UNION DISTINCT " +
-        "SELECT ID FROM PEOPLE WHERE DELETED!='Y' AND COUNTRY_CODE IN (SELECT CODE FROM COUNTRIES WHERE last_modified &gt;='${dih.last_index_time}') "
+        "SELECT ID FROM PEOPLE WHERE DELETED!='Y' AND COUNTRY_CODE IN (SELECT CODE FROM COUNTRIES WHERE last_modified &gt;='${dih.People.last_index_time}') "
         ) + "'' "
     ;
   }

Modified: lucene/dev/branches/lucene3069/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestZKPropertiesWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestZKPropertiesWriter.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestZKPropertiesWriter.java (original)
+++ lucene/dev/branches/lucene3069/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestZKPropertiesWriter.java Tue Jul 16 17:00:05 2013
@@ -60,7 +60,8 @@ public class TestZKPropertiesWriter exte
     AbstractZkTestCase.buildZooKeeper(zkServer.getZkHost(), zkServer.getZkAddress(), getFile("dih/solr"),
         "dataimport-solrconfig.xml", "dataimport-schema.xml");
 
-    initCore("dataimport-solrconfig.xml", "dataimport-schema.xml", getFile("dih/solr").getAbsolutePath());
+    //initCore("solrconfig.xml", "schema.xml", getFile("dih/solr").getAbsolutePath());
+    createDefaultCoreContainer(getFile("dih/solr").getAbsolutePath());
   }
 
   @Before

Modified: lucene/dev/branches/lucene3069/solr/contrib/extraction/ivy.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/contrib/extraction/ivy.xml?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/contrib/extraction/ivy.xml (original)
+++ lucene/dev/branches/lucene3069/solr/contrib/extraction/ivy.xml Tue Jul 16 17:00:05 2013
@@ -20,8 +20,8 @@
     <info organisation="org.apache.solr" module="extraction"/>
     <dependencies>
       <!-- Tika JARs -->
-      <dependency org="org.apache.tika" name="tika-core" rev="1.3" transitive="false"/>
-      <dependency org="org.apache.tika" name="tika-parsers" rev="1.3" transitive="false"/>
+      <dependency org="org.apache.tika" name="tika-core" rev="1.4" transitive="false"/>
+      <dependency org="org.apache.tika" name="tika-parsers" rev="1.4" transitive="false"/>
       <!-- Tika dependencies - see http://tika.apache.org/1.3/gettingstarted.html#Using_Tika_as_a_Maven_dependency -->
       <!-- When upgrading Tika, upgrade dependencies versions and add any new ones
            (except slf4j-api, commons-codec, commons-logging, geronimo-stax-api_1.0_spec) -->
@@ -31,15 +31,15 @@
       <dependency org="org.apache.james" name="apache-mime4j-core" rev="0.7.2" transitive="false"/>
       <dependency org="org.apache.james" name="apache-mime4j-dom" rev="0.7.2" transitive="false"/>
       <dependency org="org.apache.commons" name="commons-compress" rev="1.4.1" transitive="false"/>
-      <dependency org="org.apache.pdfbox" name="pdfbox" rev="1.7.1" transitive="false"/>
-      <dependency org="org.apache.pdfbox" name="fontbox" rev="1.7.1" transitive="false"/>
-      <dependency org="org.apache.pdfbox" name="jempbox" rev="1.7.1" transitive="false"/>
+      <dependency org="org.apache.pdfbox" name="pdfbox" rev="1.8.1" transitive="false"/>
+      <dependency org="org.apache.pdfbox" name="fontbox" rev="1.8.1" transitive="false"/>
+      <dependency org="org.apache.pdfbox" name="jempbox" rev="1.8.1" transitive="false"/>
       <dependency org="org.bouncycastle" name="bcmail-jdk15" rev="1.45" transitive="false"/>
       <dependency org="org.bouncycastle" name="bcprov-jdk15" rev="1.45" transitive="false"/>
-      <dependency org="org.apache.poi" name="poi" rev="3.8" transitive="false"/>
-      <dependency org="org.apache.poi" name="poi-scratchpad" rev="3.8" transitive="false"/>
-      <dependency org="org.apache.poi" name="poi-ooxml" rev="3.8" transitive="false"/>
-      <dependency org="org.apache.poi" name="poi-ooxml-schemas" rev="3.8" transitive="false"/>
+      <dependency org="org.apache.poi" name="poi" rev="3.9" transitive="false"/>
+      <dependency org="org.apache.poi" name="poi-scratchpad" rev="3.9" transitive="false"/>
+      <dependency org="org.apache.poi" name="poi-ooxml" rev="3.9" transitive="false"/>
+      <dependency org="org.apache.poi" name="poi-ooxml-schemas" rev="3.9" transitive="false"/>
       <dependency org="org.apache.xmlbeans" name="xmlbeans" rev="2.3.0" transitive="false"/>
       <dependency org="dom4j" name="dom4j" rev="1.6.1" transitive="false"/>
       <dependency org="org.ccil.cowan.tagsoup" name="tagsoup" rev="1.2.1" transitive="false"/>

Modified: lucene/dev/branches/lucene3069/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/solrconfig.xml?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/solrconfig.xml (original)
+++ lucene/dev/branches/lucene3069/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/solrconfig.xml Tue Jul 16 17:00:05 2013
@@ -20,6 +20,9 @@
 <config>
   <luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
   <jmx />
+  <indexConfig>
+    <useCompoundFile>${useCompoundFile:false}</useCompoundFile>
+  </indexConfig>
 
   <!-- Used to specify an alternate directory to hold all index data.
        It defaults to "index" if not present, and should probably
@@ -27,8 +30,6 @@
   <dataDir>${solr.data.dir:}</dataDir>
   <directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.RAMDirectoryFactory}"/>
 
-  <!-- <indexConfig> section could go here, but we want the defaults -->
-
   <updateHandler class="solr.DirectUpdateHandler2">
 
     <!-- autocommit pending docs if certain criteria are met 

Modified: lucene/dev/branches/lucene3069/solr/contrib/langid/src/java/org/apache/solr/update/processor/LangIdParams.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/contrib/langid/src/java/org/apache/solr/update/processor/LangIdParams.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/contrib/langid/src/java/org/apache/solr/update/processor/LangIdParams.java (original)
+++ lucene/dev/branches/lucene3069/solr/contrib/langid/src/java/org/apache/solr/update/processor/LangIdParams.java Tue Jul 16 17:00:05 2013
@@ -31,6 +31,7 @@ public interface LangIdParams {
   String THRESHOLD  = LANGUAGE_ID + ".threshold";            // Detection threshold
   String ENFORCE_SCHEMA =  LANGUAGE_ID + ".enforceSchema";   // Enforces that output fields exist in schema
   String LANG_WHITELIST  = LANGUAGE_ID + ".whitelist";       // Allowed languages
+  String LCMAP =  LANGUAGE_ID + ".lcmap";                    // Maps detected langcode to other value
   String MAP_ENABLE =  LANGUAGE_ID + ".map";                 // Turns on or off the field mapping
   String MAP_FL =  LANGUAGE_ID + ".map.fl";                  // Field list for mapping
   String MAP_OVERWRITE =  LANGUAGE_ID + ".map.overwrite";    // Whether to overwrite existing fields

Modified: lucene/dev/branches/lucene3069/solr/contrib/langid/src/java/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/contrib/langid/src/java/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessor.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/contrib/langid/src/java/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessor.java (original)
+++ lucene/dev/branches/lucene3069/solr/contrib/langid/src/java/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessor.java Tue Jul 16 17:00:05 2013
@@ -75,6 +75,7 @@ public abstract class LanguageIdentifier
   protected HashSet<String> mapIndividualFieldsSet;
   protected HashSet<String> allMapFieldsSet;
   protected HashMap<String,String> lcMap;
+  protected HashMap<String,String> mapLcMap;
   protected IndexSchema schema;
 
   // Regex patterns
@@ -138,13 +139,26 @@ public abstract class LanguageIdentifier
         allMapFieldsSet.addAll(mapIndividualFieldsSet);
       }
 
-      // Language Code mapping
+      // Normalize detected langcode onto normalized langcode
       lcMap = new HashMap<String,String>();
+      if(params.get(LCMAP) != null) {
+        for(String mapping : params.get(LCMAP).split("[, ]")) {
+          String[] keyVal = mapping.split(":");
+          if(keyVal.length == 2) {
+            lcMap.put(keyVal[0], keyVal[1]);
+          } else {
+            log.error("Unsupported format for langid.lcmap: "+mapping+". Skipping this mapping.");
+          }
+        }
+      }
+
+      // Language Code mapping
+      mapLcMap = new HashMap<String,String>();
       if(params.get(MAP_LCMAP) != null) {
         for(String mapping : params.get(MAP_LCMAP).split("[, ]")) {
           String[] keyVal = mapping.split(":");
           if(keyVal.length == 2) {
-            lcMap.put(keyVal[0], keyVal[1]);
+            mapLcMap.put(keyVal[0], keyVal[1]);
           } else {
             log.error("Unsupported format for langid.map.lcmap: "+mapping+". Skipping this mapping.");
           }
@@ -322,10 +336,11 @@ public abstract class LanguageIdentifier
       langStr = fallbackLang;
     } else {
       DetectedLanguage lang = languages.get(0);
-      if(langWhitelist.isEmpty() || langWhitelist.contains(lang.getLangCode())) {
-        log.debug("Language detected {} with certainty {}", lang.getLangCode(), lang.getCertainty());
+      String normalizedLang = normalizeLangCode(lang.getLangCode());
+      if(langWhitelist.isEmpty() || langWhitelist.contains(normalizedLang)) {
+        log.debug("Language detected {} with certainty {}", normalizedLang, lang.getCertainty());
         if(lang.getCertainty() >= threshold) {
-          langStr = lang.getLangCode();
+          langStr = normalizedLang;
         } else {
           log.debug("Detected language below threshold {}, using fallback {}", threshold, fallbackLang);
           langStr = fallbackLang;
@@ -345,6 +360,20 @@ public abstract class LanguageIdentifier
   }
 
   /**
+   * Looks up language code in map (langid.lcmap) and returns mapped value
+   * @param langCode the language code string returned from detector
+   * @return the normalized/mapped language code
+   */
+  protected String normalizeLangCode(String langCode) {
+    if (lcMap.containsKey(langCode)) {
+      String lc = lcMap.get(langCode);
+      log.debug("Doing langcode normalization mapping from "+langCode+" to "+lc);
+      return lc;
+    }
+    return langCode;
+  }
+
+  /**
    * Returns the name of the field to map the current contents into, so that they are properly analyzed.  For instance
    * if the currentField is "text" and the code is "en", the new field would by default be "text_en".
    * This method also performs custom regex pattern replace if configured. If enforceSchema=true
@@ -355,7 +384,7 @@ public abstract class LanguageIdentifier
    * @return The new schema field name, based on pattern and replace, or null if illegal
    */
   protected String getMappedField(String currentField, String language) {
-    String lc = lcMap.containsKey(language) ? lcMap.get(language) : language;
+    String lc = mapLcMap.containsKey(language) ? mapLcMap.get(language) : language;
     String newFieldName = langPattern.matcher(mapPattern.matcher(currentField).replaceFirst(mapReplaceStr)).replaceFirst(lc);
     if(enforceSchema && schema.getFieldOrNull(newFieldName) == null) {
       log.warn("Unsuccessful field name mapping from {} to {}, field does not exist and enforceSchema=true; skipping mapping.", currentField, newFieldName);

Modified: lucene/dev/branches/lucene3069/solr/contrib/langid/src/test-files/langid/solr/collection1/conf/solrconfig-languageidentifier.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/contrib/langid/src/test-files/langid/solr/collection1/conf/solrconfig-languageidentifier.xml?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/contrib/langid/src/test-files/langid/solr/collection1/conf/solrconfig-languageidentifier.xml (original)
+++ lucene/dev/branches/lucene3069/solr/contrib/langid/src/test-files/langid/solr/collection1/conf/solrconfig-languageidentifier.xml Tue Jul 16 17:00:05 2013
@@ -20,6 +20,9 @@
 <config>
 
   <jmx />
+  <indexConfig>
+    <useCompoundFile>${useCompoundFile:false}</useCompoundFile>
+  </indexConfig>
 
   <!-- Used to specify an alternate directory to hold all index data.
        It defaults to "index" if not present, and should probably

Modified: lucene/dev/branches/lucene3069/solr/contrib/langid/src/test/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessorFactoryTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/contrib/langid/src/test/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessorFactoryTestCase.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/contrib/langid/src/test/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessorFactoryTestCase.java (original)
+++ lucene/dev/branches/lucene3069/solr/contrib/langid/src/test/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessorFactoryTestCase.java Tue Jul 16 17:00:05 2013
@@ -116,6 +116,22 @@ public abstract class LanguageIdentifier
   }
 
   @Test
+  public void testMapLangcode() throws Exception {
+    parameters = new ModifiableSolrParams();
+    parameters.add("langid.fl", "name");
+    parameters.add("langid.lcmap", "zh_cn:zh zh_tw:zh");
+    parameters.set("langid.enforceSchema", "false");
+    liProcessor = createLangIdProcessor(parameters);
+
+    assertEquals("zh", liProcessor.resolveLanguage("zh_cn", "NA"));
+    assertEquals("zh", liProcessor.resolveLanguage("zh_tw", "NA"));
+    assertEquals("no", liProcessor.resolveLanguage("no", "NA"));
+    List<DetectedLanguage> langs = new ArrayList<DetectedLanguage>();
+    langs.add(new DetectedLanguage("zh_cn", 0.8));
+    assertEquals("zh", liProcessor.resolveLanguage(langs, "NA"));
+  }
+
+  @Test
   public void testPreExisting() throws Exception {
     SolrInputDocument doc;
     parameters = new ModifiableSolrParams();

Modified: lucene/dev/branches/lucene3069/solr/contrib/uima/src/test-files/uima/solr/collection1/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/contrib/uima/src/test-files/uima/solr/collection1/conf/solrconfig.xml?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/contrib/uima/src/test-files/uima/solr/collection1/conf/solrconfig.xml (original)
+++ lucene/dev/branches/lucene3069/solr/contrib/uima/src/test-files/uima/solr/collection1/conf/solrconfig.xml Tue Jul 16 17:00:05 2013
@@ -25,6 +25,9 @@
   -->
 <config xmlns:xi="http://www.w3.org/2001/XInclude">
   <luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
+  <indexConfig>
+    <useCompoundFile>${useCompoundFile:false}</useCompoundFile>
+  </indexConfig>
   <!--
     lib directives can be used to instruct Solr to load an Jars
     identified and use them to resolve any "plugins" specified in your
@@ -68,7 +71,6 @@
   -->
   <dataDir>${solr.data.dir:}</dataDir>
 
-  <!-- <indexConfig> section could go here, but we want the defaults -->
 
   <!--
     Enables JMX if and only if an existing MBeanServer is found, use

Modified: lucene/dev/branches/lucene3069/solr/contrib/uima/src/test-files/uima/uima-tokenizers-solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/contrib/uima/src/test-files/uima/uima-tokenizers-solrconfig.xml?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/contrib/uima/src/test-files/uima/uima-tokenizers-solrconfig.xml (original)
+++ lucene/dev/branches/lucene3069/solr/contrib/uima/src/test-files/uima/uima-tokenizers-solrconfig.xml Tue Jul 16 17:00:05 2013
@@ -25,6 +25,9 @@
   -->
 <config xmlns:xi="http://www.w3.org/2001/XInclude">
   <luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
+  <indexConfig>
+    <useCompoundFile>${useCompoundFile:false}</useCompoundFile>
+  </indexConfig>
   <!--
     lib directives can be used to instruct Solr to load an Jars
     identified and use them to resolve any "plugins" specified in your
@@ -68,8 +71,6 @@
   -->
   <dataDir>${solr.data.dir:}</dataDir>
 
-  <!-- <indexConfig> section could go here, but we want the defaults -->
-
   <!--
     Enables JMX if and only if an existing MBeanServer is found, use
     this if you want to configure JMX through JVM parameters. Remove

Modified: lucene/dev/branches/lucene3069/solr/contrib/velocity/src/java/org/apache/solr/response/VelocityResponseWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/contrib/velocity/src/java/org/apache/solr/response/VelocityResponseWriter.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/contrib/velocity/src/java/org/apache/solr/response/VelocityResponseWriter.java (original)
+++ lucene/dev/branches/lucene3069/solr/contrib/velocity/src/java/org/apache/solr/response/VelocityResponseWriter.java Tue Jul 16 17:00:05 2013
@@ -17,6 +17,7 @@
 
 package org.apache.solr.response;
 
+import org.apache.lucene.util.IOUtils;
 import org.apache.solr.client.solrj.SolrResponse;
 import org.apache.solr.client.solrj.response.QueryResponse;
 import org.apache.solr.client.solrj.response.SolrResponseBase;
@@ -150,7 +151,7 @@ public class VelocityResponseWriter impl
         try {
           is = resourceLoader.getResourceStream(propFile);
           Properties props = new Properties();
-          props.load(is);
+          props.load(new InputStreamReader(is, IOUtils.CHARSET_UTF_8));
           engine.init(props);
         }
         finally {

Modified: lucene/dev/branches/lucene3069/solr/contrib/velocity/src/test-files/velocity/solr/collection1/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/contrib/velocity/src/test-files/velocity/solr/collection1/conf/solrconfig.xml?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/contrib/velocity/src/test-files/velocity/solr/collection1/conf/solrconfig.xml (original)
+++ lucene/dev/branches/lucene3069/solr/contrib/velocity/src/test-files/velocity/solr/collection1/conf/solrconfig.xml Tue Jul 16 17:00:05 2013
@@ -22,6 +22,9 @@
 -->
 <config>
   <luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
+  <indexConfig>
+    <useCompoundFile>${useCompoundFile:false}</useCompoundFile>
+  </indexConfig>
 
   <lib dir="../../contrib/velocity/lib" />
   <lib dir="../../dist/" regex="solr-velocity-\d.*\.jar" />
@@ -31,9 +34,6 @@
   <directoryFactory name="DirectoryFactory"
                     class="${solr.directoryFactory:solr.StandardDirectoryFactory}"/>
 
-
-  <!-- <indexConfig> section could go here, but we want the defaults -->
-
   <updateHandler class="solr.DirectUpdateHandler2">
   </updateHandler>
   

Modified: lucene/dev/branches/lucene3069/solr/core/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/core/build.xml?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/core/build.xml (original)
+++ lucene/dev/branches/lucene3069/solr/core/build.xml Tue Jul 16 17:00:05 2013
@@ -15,16 +15,23 @@
     See the License for the specific language governing permissions and
     limitations under the License.
  -->
-<project name="solr-core" default="default">
+<project name="solr-core" default="default" xmlns:ivy="antlib:org.apache.ivy.ant">
   <description>Solr Core</description>
 
   <!-- html file for testing -->
   <property name="rat.excludes" value="**/htmlStripReaderTest.html,**/*.iml"/>
+  
+  <property name="test.lib.dir" location="test-lib"/>
 
   <import file="../common-build.xml"/>
 
   <target name="compile-core" depends="compile-solrj,common-solr.compile-core"/>
 
+  <path id="test.classpath">
+    <path refid="solr.test.base.classpath"/>
+    <fileset dir="${test.lib.dir}" includes="*.jar"/>
+  </path>
+
   <!-- specialized to ONLY depend on solrj -->
   <target name="javadocs" depends="compile-core,define-lucene-javadoc-url,lucene-javadocs,javadocs-solrj">
     <sequential>
@@ -43,8 +50,13 @@
 
   <target name="dist-maven" depends="dist-maven-src-java"/>
 
-
-
+  <target name="resolve" depends="ivy-availability-check,ivy-fail,ivy-configure">
+    <sequential>
+      <ivy:retrieve conf="compile,compile.hadoop" type="jar,bundle" sync="${ivy.sync}" log="download-only"/>
+      <ivy:retrieve conf="test,test.DfsMiniCluster" type="jar,bundle,test" sync="${ivy.sync}" log="download-only"
+                    pattern="${test.lib.dir}/[artifact]-[revision](-[classifier]).[ext]"/>
+    </sequential>
+  </target>
 
   <target name="javacc" depends="javacc-QueryParser"/>
   <target name="javacc-QueryParser" depends="resolve-javacc">
@@ -67,7 +79,7 @@
   <target name="resolve-javacc" xmlns:ivy="antlib:org.apache.ivy.ant">
     <!-- setup a "fake" JavaCC distribution folder in ${build.dir} to make JavaCC ANT task happy: -->
     <ivy:retrieve organisation="net.java.dev.javacc" module="javacc" revision="5.0"
-      inline="true" conf="default" transitive="false" type="jar" sync="true"
+      inline="true" transitive="false" type="jar" sync="true"
       pattern="${build.dir}/javacc/bin/lib/[artifact].[ext]"/>
   </target>
 

Modified: lucene/dev/branches/lucene3069/solr/core/ivy.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/core/ivy.xml?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/core/ivy.xml (original)
+++ lucene/dev/branches/lucene3069/solr/core/ivy.xml Tue Jul 16 17:00:05 2013
@@ -16,22 +16,57 @@
    specific language governing permissions and limitations
    under the License.    
 -->
-<ivy-module version="2.0">
-    <info organisation="org.apache.solr" module="core"/>
+<!DOCTYPE ivy-module [
+  <!ENTITY hadoop.version "2.0.5-alpha">
+]>
+<ivy-module version="2.0"  xmlns:maven="http://ant.apache.org/ivy/maven">
+  <info organisation="org.apache.solr" module="core"/>
+  
+  <configurations>
+    <!-- artifacts in the "compile" and "compile.hadoop" configurations will go into solr/core/lib/ -->
+    <conf name="compile" transitive="false"/>
+    <conf name="test" transitive="false"/>
+    <conf name="compile.hadoop" transitive="false"/>
+    <!-- artifacts in the "test" and "test.DfsMiniCluster" configuration will go into solr/core/test-lib/ -->
+    <conf name="test.DfsMiniCluster" transitive="false"/>
+  </configurations>
 
-    <dependencies>
-      <dependency org="commons-codec" name="commons-codec" rev="1.7" transitive="false"/>
-      <dependency org="commons-fileupload" name="commons-fileupload" rev="1.2.1" transitive="false"/>
-      <dependency org="commons-cli" name="commons-cli" rev="1.2" transitive="false"/>
-      <dependency org="commons-lang" name="commons-lang" rev="2.6" transitive="false"/>
-      <dependency org="com.google.guava" name="guava" rev="14.0.1" transitive="false"/>
-      <dependency org="org.easymock" name="easymock" rev="3.0" transitive="false"/>
-      <dependency org="cglib" name="cglib-nodep" rev="2.2" transitive="false"/>
-      <dependency org="org.objenesis" name="objenesis" rev="1.2" transitive="false"/>
-      <dependency org="com.spatial4j" name="spatial4j" rev="0.3" transitive="false"/>
-      <dependency org="javax.servlet" name="javax.servlet-api" rev="3.0.1" transitive="false"/>
-      <dependency org="org.restlet.jee" name="org.restlet" rev="2.1.1" transitive="false"/>
-      <dependency org="org.restlet.jee" name="org.restlet.ext.servlet" rev="2.1.1" transitive="false"/>
-      <exclude org="*" ext="*" matcher="regexp" type="${ivy.exclude.types}"/>
-    </dependencies>
+  <dependencies>
+    <dependency org="commons-codec" name="commons-codec" rev="1.7" conf="compile->*"/>
+    <dependency org="commons-fileupload" name="commons-fileupload" rev="1.2.1" conf="compile->*"/>
+    <dependency org="commons-cli" name="commons-cli" rev="1.2" conf="compile->*"/>
+    <dependency org="commons-lang" name="commons-lang" rev="2.6" conf="compile->*"/>
+    <dependency org="com.google.guava" name="guava" rev="14.0.1" conf="compile->*"/>
+    <dependency org="com.spatial4j" name="spatial4j" rev="0.3" conf="compile->*"/>
+    <dependency org="org.restlet.jee" name="org.restlet" rev="2.1.1" conf="compile->*"/>
+    <dependency org="org.restlet.jee" name="org.restlet.ext.servlet" rev="2.1.1" conf="compile->*"/>
+    <dependency org="joda-time" name="joda-time" rev="2.2" conf="compile->*"/>
+
+    <dependency org="javax.servlet" name="javax.servlet-api" rev="3.0.1" conf="test->*"/>
+    <dependency org="org.easymock" name="easymock" rev="3.0" conf="test->*"/>
+    <dependency org="cglib" name="cglib-nodep" rev="2.2" conf="test->*"/>
+    <dependency org="org.objenesis" name="objenesis" rev="1.2" conf="test->*"/>
+
+    <dependency org="org.apache.hadoop" name="hadoop-common" rev="&hadoop.version;" conf="compile.hadoop->*"/>
+    <dependency org="org.apache.hadoop" name="hadoop-hdfs" rev="&hadoop.version;" conf="compile.hadoop->*"/>
+    <dependency org="org.apache.hadoop" name="hadoop-annotations" rev="&hadoop.version;" conf="compile.hadoop->*"/>
+    <dependency org="org.apache.hadoop" name="hadoop-auth" rev="&hadoop.version;" conf="compile.hadoop->*"/>
+    <dependency org="commons-configuration" name="commons-configuration" rev="1.6" conf="compile.hadoop->*"/>
+    <dependency org="com.google.protobuf" name="protobuf-java" rev="2.4.0a" conf="compile.hadoop->*"/>
+    <dependency org="com.googlecode.concurrentlinkedhashmap" name="concurrentlinkedhashmap-lru" rev="1.2" conf="compile.hadoop->*"/>
+
+    <!-- Hadoop DfsMiniCluster Dependencies-->
+    <dependency org="org.apache.hadoop" name="hadoop-common" rev="&hadoop.version;" conf="test.DfsMiniCluster->*">
+      <artifact name="hadoop-common" type="test" ext="jar" maven:classifier="tests" />
+    </dependency>
+    <dependency org="org.apache.hadoop" name="hadoop-hdfs" rev="&hadoop.version;" conf="test.DfsMiniCluster->*">
+      <artifact name="hadoop-hdfs" type="test" ext="jar" maven:classifier="tests" />
+    </dependency>
+    <dependency org="org.mortbay.jetty" name="jetty" rev="6.1.26" conf="test.DfsMiniCluster->*"/>
+    <dependency org="org.mortbay.jetty" name="jetty-util" rev="6.1.26" conf="test.DfsMiniCluster->*"/>
+    <dependency org="com.sun.jersey" name="jersey-core" rev="1.16" conf="test.DfsMiniCluster->*"/>
+    <dependency org="commons-collections" name="commons-collections" rev="3.2.1" conf="test.DfsMiniCluster->*"/>
+
+    <exclude org="*" ext="*" matcher="regexp" type="${ivy.exclude.types}"/> 
+  </dependencies>
 </ivy-module>

Modified: lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/SolrLogFormatter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/SolrLogFormatter.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/SolrLogFormatter.java (original)
+++ lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/SolrLogFormatter.java Tue Jul 16 17:00:05 2013
@@ -17,21 +17,25 @@ package org.apache.solr;
  * limitations under the License.
  */
 
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.WeakHashMap;
+import java.util.logging.ConsoleHandler;
+import java.util.logging.Formatter;
+import java.util.logging.Handler;
+import java.util.logging.Level;
+import java.util.logging.LogRecord;
+import java.util.logging.Logger;
+
 import org.apache.solr.cloud.ZkController;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.cloud.Replica;
-import org.apache.solr.common.cloud.ZkStateReader;
 import org.apache.solr.core.SolrCore;
 import org.apache.solr.request.SolrQueryRequest;
 import org.apache.solr.request.SolrRequestInfo;
 import org.slf4j.LoggerFactory;
 
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.WeakHashMap;
-import java.util.logging.*;
-
 public class SolrLogFormatter extends Formatter {
 
   /** Add this interface to a thread group and the string returned by
@@ -259,7 +263,7 @@ sb.append("(group_name=").append(tg.getN
 
   private Map<String,Object> getReplicaProps(ZkController zkController, SolrCore core) {
     final String collection = core.getCoreDescriptor().getCloudDescriptor().getCollectionName();
-    Replica replica = zkController.getClusterState().getReplica(collection, zkController.getCoreNodeName(core.getCoreDescriptor()));
+    Replica replica = zkController.getClusterState().getReplica(collection, core.getCoreDescriptor().getCloudDescriptor().getCoreNodeName());
     if(replica!=null) {
       return replica.getProperties();
     }

Modified: lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java (original)
+++ lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java Tue Jul 16 17:00:05 2013
@@ -85,6 +85,7 @@ public class JettySolrRunner {
   private String shards;
 
   private String dataDir;
+  private String solrUlogDir;
   
   private volatile boolean startedBefore = false;
 
@@ -359,6 +360,9 @@ public class JettySolrRunner {
     if( dataDir != null) {
       System.setProperty("solr.data.dir", dataDir);
     }
+    if( solrUlogDir != null) {
+      System.setProperty("solr.ulog.dir", solrUlogDir);
+    }
     if(shards != null) {
       System.setProperty("shard", shards);
     }
@@ -382,6 +386,8 @@ public class JettySolrRunner {
     System.clearProperty("shard");
     System.clearProperty("solr.data.dir");
     System.clearProperty("coreNodeName");
+    System.clearProperty("solr.ulog.dir");
+
   }
 
   public void stop() throws Exception {
@@ -485,6 +491,10 @@ public class JettySolrRunner {
   public void setDataDir(String dataDir) {
     this.dataDir = dataDir;
   }
+  
+  public void setUlogDir(String ulogDir) {
+    this.solrUlogDir = ulogDir;
+  }
 
   public void setCoreNodeName(String coreNodeName) {
     this.coreNodeName = coreNodeName;

Modified: lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/cloud/CloudDescriptor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/cloud/CloudDescriptor.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/cloud/CloudDescriptor.java (original)
+++ lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/cloud/CloudDescriptor.java Tue Jul 16 17:00:05 2013
@@ -17,11 +17,16 @@ package org.apache.solr.cloud;
  * limitations under the License.
  */
 
-import org.apache.solr.common.cloud.ZkStateReader;
 import org.apache.solr.common.cloud.Slice;
+import org.apache.solr.common.cloud.ZkStateReader;
 import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.core.CoreDescriptor;
+import org.apache.solr.util.PropertiesUtil;
+
+import java.util.Properties;
 
 public class CloudDescriptor {
+
   private String shardId;
   private String collectionName;
   private SolrParams params;
@@ -36,6 +41,21 @@ public class CloudDescriptor {
 
   volatile boolean isLeader = false;
   volatile String lastPublished = ZkStateReader.ACTIVE;
+
+  public static final String SHARD_STATE = "shardState";
+  public static final String NUM_SHARDS = "numShards";
+  public static final String SHARD_RANGE = "shardRange";
+
+  public CloudDescriptor(String coreName, Properties props) {
+    this.shardId = props.getProperty(CoreDescriptor.CORE_SHARD, null);
+    // If no collection name is specified, we default to the core name
+    this.collectionName = props.getProperty(CoreDescriptor.CORE_COLLECTION, coreName);
+    this.roles = props.getProperty(CoreDescriptor.CORE_ROLES, null);
+    this.nodeName = props.getProperty(CoreDescriptor.CORE_NODE_NAME);
+    this.shardState = props.getProperty(CloudDescriptor.SHARD_STATE, Slice.ACTIVE);
+    this.numShards = PropertiesUtil.toInteger(props.getProperty(CloudDescriptor.NUM_SHARDS), null);
+    this.shardRange = props.getProperty(CloudDescriptor.SHARD_RANGE, null);
+  }
   
   public String getLastPublished() {
     return lastPublished;
@@ -44,6 +64,10 @@ public class CloudDescriptor {
   public boolean isLeader() {
     return isLeader;
   }
+  
+  public void setLeader(boolean isLeader) {
+    this.isLeader = isLeader;
+  }
 
   public void setShardId(String shardId) {
     this.shardId = shardId;

Modified: lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java (original)
+++ lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java Tue Jul 16 17:00:05 2013
@@ -47,9 +47,9 @@ public abstract class ElectionContext {
   String leaderSeqPath;
   private SolrZkClient zkClient;
   
-  public ElectionContext(final String shardZkNodeName,
+  public ElectionContext(final String coreNodeName,
       final String electionPath, final String leaderPath, final ZkNodeProps leaderProps, final SolrZkClient zkClient) {
-    this.id = shardZkNodeName;
+    this.id = coreNodeName;
     this.electionPath = electionPath;
     this.leaderPath = leaderPath;
     this.leaderProps = leaderProps;
@@ -78,8 +78,8 @@ class ShardLeaderElectionContextBase ext
   protected LeaderElector leaderElector;
 
   public ShardLeaderElectionContextBase(LeaderElector leaderElector, final String shardId,
-      final String collection, final String shardZkNodeName, ZkNodeProps props, ZkStateReader zkStateReader) {
-    super(shardZkNodeName, ZkStateReader.COLLECTIONS_ZKNODE + "/" + collection + "/leader_elect/"
+      final String collection, final String coreNodeName, ZkNodeProps props, ZkStateReader zkStateReader) {
+    super(coreNodeName, ZkStateReader.COLLECTIONS_ZKNODE + "/" + collection + "/leader_elect/"
         + shardId, ZkStateReader.getShardLeadersPath(collection, shardId),
         props, zkStateReader.getZkClient());
     this.leaderElector = leaderElector;
@@ -95,7 +95,7 @@ class ShardLeaderElectionContextBase ext
     zkClient.makePath(leaderPath, ZkStateReader.toJSON(leaderProps),
         CreateMode.EPHEMERAL, true);
     assert shardId != null;
-    ZkNodeProps m = ZkNodeProps.fromKeyVals(Overseer.QUEUE_OPERATION, "leader",
+    ZkNodeProps m = ZkNodeProps.fromKeyVals(Overseer.QUEUE_OPERATION, ZkStateReader.LEADER_PROP,
         ZkStateReader.SHARD_ID_PROP, shardId, ZkStateReader.COLLECTION_PROP,
         collection, ZkStateReader.BASE_URL_PROP, leaderProps.getProperties()
             .get(ZkStateReader.BASE_URL_PROP), ZkStateReader.CORE_NAME_PROP,
@@ -119,8 +119,8 @@ final class ShardLeaderElectionContext e
   
   public ShardLeaderElectionContext(LeaderElector leaderElector, 
       final String shardId, final String collection,
-      final String shardZkNodeName, ZkNodeProps props, ZkController zkController, CoreContainer cc) {
-    super(leaderElector, shardId, collection, shardZkNodeName, props,
+      final String coreNodeName, ZkNodeProps props, ZkController zkController, CoreContainer cc) {
+    super(leaderElector, shardId, collection, coreNodeName, props,
         zkController.getZkStateReader());
     this.zkController = zkController;
     this.cc = cc;
@@ -138,12 +138,12 @@ final class ShardLeaderElectionContext e
   @Override
   void runLeaderProcess(boolean weAreReplacement) throws KeeperException,
       InterruptedException, IOException {
-    log.info("Running the leader process.");
+    log.info("Running the leader process for shard " + shardId);
     
     String coreName = leaderProps.getStr(ZkStateReader.CORE_NAME_PROP);
     
     // clear the leader in clusterstate
-    ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION, "leader",
+    ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION, ZkStateReader.LEADER_PROP,
         ZkStateReader.SHARD_ID_PROP, shardId, ZkStateReader.COLLECTION_PROP,
         collection);
     Overseer.getInQueue(zkClient).offer(ZkStateReader.toJSON(m));
@@ -243,8 +243,8 @@ final class ShardLeaderElectionContext e
       }
 
       log.info("I am the new leader: "
-          + ZkCoreNodeProps.getCoreUrl(leaderProps));
-      core.getCoreDescriptor().getCloudDescriptor().isLeader = true;
+          + ZkCoreNodeProps.getCoreUrl(leaderProps) + " " + shardId);
+      core.getCoreDescriptor().getCloudDescriptor().setLeader(true);
     } finally {
       if (core != null) {
         core.close();
@@ -254,16 +254,17 @@ final class ShardLeaderElectionContext e
     try {
       super.runLeaderProcess(weAreReplacement);
     } catch (Throwable t) {
+      SolrException.log(log, "There was a problem trying to register as the leader", t);
+      cancelElection();
       try {
         core = cc.getCore(coreName);
         if (core == null) {
-          cancelElection();
           throw new SolrException(ErrorCode.SERVER_ERROR,
               "Fatal Error, SolrCore not found:" + coreName + " in "
                   + cc.getCoreNames());
         }
         
-        core.getCoreDescriptor().getCloudDescriptor().isLeader = false;
+        core.getCoreDescriptor().getCloudDescriptor().setLeader(false);
         
         // we could not publish ourselves as leader - rejoin election
         rejoinLeaderElection(leaderSeqPath, core);
@@ -332,7 +333,7 @@ final class ShardLeaderElectionContext e
           return;
         } else {
           if (cnt % 40 == 0) {
-            log.info("Waiting until we see more replicas up: total="
+            log.info("Waiting until we see more replicas up for shard " + shardId + ": total="
               + slices.getReplicasMap().size() + " found=" + found
               + " timeoutin=" + (timeoutAt - System.currentTimeMillis()));
           }



Mime
View raw message