Return-Path: Delivered-To: apmail-lucene-solr-commits-archive@minotaur.apache.org Received: (qmail 10846 invoked from network); 30 Sep 2009 10:35:54 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.3) by minotaur.apache.org with SMTP; 30 Sep 2009 10:35:54 -0000 Received: (qmail 27677 invoked by uid 500); 30 Sep 2009 10:35:54 -0000 Delivered-To: apmail-lucene-solr-commits-archive@lucene.apache.org Received: (qmail 27626 invoked by uid 500); 30 Sep 2009 10:35:53 -0000 Mailing-List: contact solr-commits-help@lucene.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: solr-dev@lucene.apache.org Delivered-To: mailing list solr-commits@lucene.apache.org Received: (qmail 27617 invoked by uid 99); 30 Sep 2009 10:35:53 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 30 Sep 2009 10:35:53 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 30 Sep 2009 10:35:44 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 8CACD23888D7; Wed, 30 Sep 2009 10:35:24 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r820235 - in /lucene/solr/trunk/contrib/dataimporthandler: ./ src/main/java/org/apache/solr/handler/dataimport/ Date: Wed, 30 Sep 2009 10:35:24 -0000 To: solr-commits@lucene.apache.org From: shalin@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20090930103524.8CACD23888D7@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: shalin Date: Wed Sep 30 10:35:23 2009 New Revision: 820235 URL: http://svn.apache.org/viewvc?rev=820235&view=rev Log: SOLR-1474 -- Delta-import should run even if last_index_time is not set Modified: lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTest.java lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/SolrWriter.java Modified: lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt?rev=820235&r1=820234&r2=820235&view=diff ============================================================================== --- lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt (original) +++ lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt Wed Sep 30 10:35:23 2009 @@ -29,6 +29,10 @@ parent-row at the end of row (end of data). A new method EntityProcessor#close is added which is called at the end of import. +In Solr 1.3, if the last_index_time was not available (first import) and a delta-import was requested, a full-import +was run instead. This is no longer the case. In Solr 1.4 delta import is run with last_index_time as the epoch +date (January 1, 1970, 00:00:00 GMT) if last_index_time is not available. + Detailed Change List ---------------------- @@ -265,6 +269,9 @@ 30.SOLR-1450: Jdbc connection properties such as batchSize are not applied if the driver jar is placed in solr_home/lib. (Steve Sun via shalin) + +31.SOLR-1474: Delta-import should run even if last_index_time is not set. + (shalin) Documentation Modified: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTest.java URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTest.java?rev=820235&r1=820234&r2=820235&view=diff ============================================================================== --- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTest.java (original) +++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTest.java Wed Sep 30 10:35:23 2009 @@ -22,6 +22,7 @@ import org.apache.solr.common.util.NamedList; import java.io.IOException; +import java.io.File; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -46,6 +47,14 @@ @Override public void tearDown() throws Exception { + // remove dataimport.properties + File f = new File("solr/conf/dataimport.properties"); + log.info("Looking for dataimport.properties at: " + f.getAbsolutePath()); + if (f.exists()) { + log.info("Deleting dataimport.properties"); + if (!f.delete()) + log.warn("Could not delete dataimport.properties"); + } super.tearDown(); } Modified: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java?rev=820235&r1=820234&r2=820235&view=diff ============================================================================== --- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java (original) +++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java Wed Sep 30 10:35:23 2009 @@ -252,14 +252,6 @@ this.indexStartTime = indextStartTime; } - Date getLastIndexTime() { - return lastIndexTime; - } - - void setLastIndexTime(Date lastIndexTime) { - this.lastIndexTime = lastIndexTime; - } - void store(Object key, Object value) { store.put(key, value); } @@ -387,8 +379,6 @@ return; } try { - Date lastModified = sw.loadIndexStartTime(); - setLastIndexTime(lastModified); if (FULL_IMPORT_CMD.equals(command) || IMPORT_CMD.equals(command)) { doFullImport(sw, reqParams); } else if (command.equals(DELTA_IMPORT_CMD)) { Modified: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java?rev=820235&r1=820234&r2=820235&view=diff ============================================================================== --- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java (original) +++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java Wed Sep 30 10:35:23 2009 @@ -27,6 +27,7 @@ import java.util.*; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; +import java.text.ParseException; /** *

DocBuilder is responsible for creating Solr documents out of the given configuration. It also maintains @@ -41,6 +42,8 @@ private static final Logger LOG = LoggerFactory.getLogger(DocBuilder.class); + private static final Date EPOCH = new Date(0); + DataImporter dataImporter; private DataConfig.Document document; @@ -76,24 +79,36 @@ } public VariableResolverImpl getVariableResolver() { - VariableResolverImpl resolver = new VariableResolverImpl(); - Map indexerNamespace = new HashMap(); - if (dataImporter.getLastIndexTime() != null) - indexerNamespace.put(LAST_INDEX_TIME, - DataImporter.DATE_TIME_FORMAT.get().format(dataImporter.getLastIndexTime())); - indexerNamespace.put(INDEX_START_TIME, dataImporter.getIndexStartTime()); - indexerNamespace.put("request", requestParameters.requestParams); - indexerNamespace.put("functions", functionsNamespace); - for (DataConfig.Entity entity : dataImporter.getConfig().document.entities) { - String key = entity.name + "." + SolrWriter.LAST_INDEX_KEY; - String lastIndex = persistedProperties.getProperty(key); - if (lastIndex != null) { - indexerNamespace.put(key, lastIndex); - } - } - resolver.addNamespace(DataConfig.IMPORTER_NS_SHORT, indexerNamespace); - resolver.addNamespace(DataConfig.IMPORTER_NS, indexerNamespace); - return resolver; + try { + VariableResolverImpl resolver = new VariableResolverImpl(); + Map indexerNamespace = new HashMap(); + if (persistedProperties.getProperty(LAST_INDEX_TIME) != null) { + indexerNamespace.put(LAST_INDEX_TIME, + DataImporter.DATE_TIME_FORMAT.get().parse(persistedProperties.getProperty(LAST_INDEX_TIME))); + } else { + // set epoch + indexerNamespace.put(LAST_INDEX_TIME, EPOCH); + } + indexerNamespace.put(INDEX_START_TIME, dataImporter.getIndexStartTime()); + indexerNamespace.put("request", requestParameters.requestParams); + indexerNamespace.put("functions", functionsNamespace); + for (DataConfig.Entity entity : dataImporter.getConfig().document.entities) { + String key = entity.name + "." + SolrWriter.LAST_INDEX_KEY; + String lastIndex = persistedProperties.getProperty(key); + if (lastIndex != null) { + indexerNamespace.put(key, lastIndex); + } else { + indexerNamespace.put(key, EPOCH); + } + } + resolver.addNamespace(DataConfig.IMPORTER_NS_SHORT, indexerNamespace); + resolver.addNamespace(DataConfig.IMPORTER_NS, indexerNamespace); + return resolver; + } catch (ParseException e) { + DataImportHandlerException.wrapAndThrow(DataImportHandlerException.SEVERE, e); + // unreachable statement + return null; + } } private void invokeEventListener(String className) { @@ -153,8 +168,7 @@ DataImporter.DATE_TIME_FORMAT.get().format(new Date())); root = e; String delQuery = e.allAttributes.get("preImportDeleteQuery"); - if (dataImporter.getStatus() == DataImporter.Status.RUNNING_DELTA_DUMP - && dataImporter.getLastIndexTime() != null) { + if (dataImporter.getStatus() == DataImporter.Status.RUNNING_DELTA_DUMP) { cleanByQuery(delQuery, fullCleanDone); doDelta(); delQuery = e.allAttributes.get("postImportDeleteQuery"); Modified: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/SolrWriter.java URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/SolrWriter.java?rev=820235&r1=820234&r2=820235&view=diff ============================================================================== --- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/SolrWriter.java (original) +++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/SolrWriter.java Wed Sep 30 10:35:23 2009 @@ -219,22 +219,6 @@ } } - public Date loadIndexStartTime() { - Properties props; - props = readIndexerProperties(); - String result = props.getProperty(SolrWriter.LAST_INDEX_KEY); - - try { - if (result != null) - return DataImporter.DATE_TIME_FORMAT.get().parse(result); - } catch (ParseException e) { - throw new DataImportHandlerException(DataImportHandlerException.WARN, - "Unable to read last indexed time from: " - + persistFilename, e); - } - return null; - } - public DebugLogger getDebugLogger() { if (debugLogger == null) { debugLogger = new DebugLogger(this);