lucene-solr-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sha...@apache.org
Subject svn commit: r820235 - in /lucene/solr/trunk/contrib/dataimporthandler: ./ src/main/java/org/apache/solr/handler/dataimport/
Date Wed, 30 Sep 2009 10:35:24 GMT
Author: shalin
Date: Wed Sep 30 10:35:23 2009
New Revision: 820235

URL: http://svn.apache.org/viewvc?rev=820235&view=rev
Log:
SOLR-1474 -- Delta-import should run even if last_index_time is not set

Modified:
    lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt
    lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTest.java
    lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java
    lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java
    lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/SolrWriter.java

Modified: lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt?rev=820235&r1=820234&r2=820235&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt Wed Sep 30 10:35:23 2009
@@ -29,6 +29,10 @@
 parent-row at the end of row (end of data). A new method EntityProcessor#close is added which
is called at the end
 of import.
 
+In Solr 1.3, if the last_index_time was not available (first import) and a delta-import was
requested, a full-import
+was run instead. This is no longer the case. In Solr 1.4 delta import is run with last_index_time
as the epoch
+date (January 1, 1970, 00:00:00 GMT) if last_index_time is not available.
+
 Detailed Change List
 ----------------------
 
@@ -265,6 +269,9 @@
 30.SOLR-1450: Jdbc connection properties such as batchSize are not applied if the driver
jar is placed
               in solr_home/lib.
               (Steve Sun via shalin)
+
+31.SOLR-1474: Delta-import should run even if last_index_time is not set.
+              (shalin)
              
 
 Documentation

Modified: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTest.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTest.java?rev=820235&r1=820234&r2=820235&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTest.java
(original)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTest.java
Wed Sep 30 10:35:23 2009
@@ -22,6 +22,7 @@
 import org.apache.solr.common.util.NamedList;
 
 import java.io.IOException;
+import java.io.File;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -46,6 +47,14 @@
 
   @Override
   public void tearDown() throws Exception {
+    // remove dataimport.properties
+    File f = new File("solr/conf/dataimport.properties");
+    log.info("Looking for dataimport.properties at: " + f.getAbsolutePath());
+    if (f.exists()) {
+      log.info("Deleting dataimport.properties");
+      if (!f.delete())
+        log.warn("Could not delete dataimport.properties");
+    }
     super.tearDown();
   }
 

Modified: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java?rev=820235&r1=820234&r2=820235&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java
(original)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java
Wed Sep 30 10:35:23 2009
@@ -252,14 +252,6 @@
     this.indexStartTime = indextStartTime;
   }
 
-  Date getLastIndexTime() {
-    return lastIndexTime;
-  }
-
-  void setLastIndexTime(Date lastIndexTime) {
-    this.lastIndexTime = lastIndexTime;
-  }
-
   void store(Object key, Object value) {
     store.put(key, value);
   }
@@ -387,8 +379,6 @@
       return;
     }
     try {
-      Date lastModified = sw.loadIndexStartTime();
-      setLastIndexTime(lastModified);
       if (FULL_IMPORT_CMD.equals(command) || IMPORT_CMD.equals(command)) {
         doFullImport(sw, reqParams);
       } else if (command.equals(DELTA_IMPORT_CMD)) {

Modified: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java?rev=820235&r1=820234&r2=820235&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java
(original)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java
Wed Sep 30 10:35:23 2009
@@ -27,6 +27,7 @@
 import java.util.*;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
+import java.text.ParseException;
 
 /**
  * <p> DocBuilder is responsible for creating Solr documents out of the given configuration.
It also maintains
@@ -41,6 +42,8 @@
 
   private static final Logger LOG = LoggerFactory.getLogger(DocBuilder.class);
 
+  private static final Date EPOCH = new Date(0);
+
   DataImporter dataImporter;
 
   private DataConfig.Document document;
@@ -76,24 +79,36 @@
   }
 
   public VariableResolverImpl getVariableResolver() {
-    VariableResolverImpl resolver = new VariableResolverImpl();
-    Map<String, Object> indexerNamespace = new HashMap<String, Object>();
-    if (dataImporter.getLastIndexTime() != null)
-      indexerNamespace.put(LAST_INDEX_TIME,
-              DataImporter.DATE_TIME_FORMAT.get().format(dataImporter.getLastIndexTime()));
-    indexerNamespace.put(INDEX_START_TIME, dataImporter.getIndexStartTime());
-    indexerNamespace.put("request", requestParameters.requestParams);
-    indexerNamespace.put("functions", functionsNamespace);
-    for (DataConfig.Entity entity : dataImporter.getConfig().document.entities) {
-      String key = entity.name + "." + SolrWriter.LAST_INDEX_KEY;
-      String lastIndex = persistedProperties.getProperty(key);
-      if (lastIndex != null) {
-        indexerNamespace.put(key, lastIndex);
-      }
-    }
-    resolver.addNamespace(DataConfig.IMPORTER_NS_SHORT, indexerNamespace);
-    resolver.addNamespace(DataConfig.IMPORTER_NS, indexerNamespace);
-    return resolver;
+    try {
+      VariableResolverImpl resolver = new VariableResolverImpl();
+      Map<String, Object> indexerNamespace = new HashMap<String, Object>();
+      if (persistedProperties.getProperty(LAST_INDEX_TIME) != null) {
+        indexerNamespace.put(LAST_INDEX_TIME,
+                DataImporter.DATE_TIME_FORMAT.get().parse(persistedProperties.getProperty(LAST_INDEX_TIME)));
+      } else  {
+        // set epoch
+        indexerNamespace.put(LAST_INDEX_TIME, EPOCH);
+      }
+      indexerNamespace.put(INDEX_START_TIME, dataImporter.getIndexStartTime());
+      indexerNamespace.put("request", requestParameters.requestParams);
+      indexerNamespace.put("functions", functionsNamespace);
+      for (DataConfig.Entity entity : dataImporter.getConfig().document.entities) {
+        String key = entity.name + "." + SolrWriter.LAST_INDEX_KEY;
+        String lastIndex = persistedProperties.getProperty(key);
+        if (lastIndex != null) {
+          indexerNamespace.put(key, lastIndex);
+        } else  {
+          indexerNamespace.put(key, EPOCH);
+        }
+      }
+      resolver.addNamespace(DataConfig.IMPORTER_NS_SHORT, indexerNamespace);
+      resolver.addNamespace(DataConfig.IMPORTER_NS, indexerNamespace);
+      return resolver;
+    } catch (ParseException e) {
+      DataImportHandlerException.wrapAndThrow(DataImportHandlerException.SEVERE, e);
+      // unreachable statement
+      return null;
+    }
   }
 
   private void invokeEventListener(String className) {
@@ -153,8 +168,7 @@
               DataImporter.DATE_TIME_FORMAT.get().format(new Date()));
       root = e;
       String delQuery = e.allAttributes.get("preImportDeleteQuery");
-      if (dataImporter.getStatus() == DataImporter.Status.RUNNING_DELTA_DUMP
-              && dataImporter.getLastIndexTime() != null) {
+      if (dataImporter.getStatus() == DataImporter.Status.RUNNING_DELTA_DUMP) {
         cleanByQuery(delQuery, fullCleanDone);
         doDelta();
         delQuery = e.allAttributes.get("postImportDeleteQuery");

Modified: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/SolrWriter.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/SolrWriter.java?rev=820235&r1=820234&r2=820235&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/SolrWriter.java
(original)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/SolrWriter.java
Wed Sep 30 10:35:23 2009
@@ -219,22 +219,6 @@
     }
   }
 
-  public Date loadIndexStartTime() {
-    Properties props;
-    props = readIndexerProperties();
-    String result = props.getProperty(SolrWriter.LAST_INDEX_KEY);
-
-    try {
-      if (result != null)
-        return DataImporter.DATE_TIME_FORMAT.get().parse(result);
-    } catch (ParseException e) {
-      throw new DataImportHandlerException(DataImportHandlerException.WARN,
-              "Unable to read last indexed time from: "
-                      + persistFilename, e);
-    }
-    return null;
-  }
-
   public DebugLogger getDebugLogger() {
     if (debugLogger == null) {
       debugLogger = new DebugLogger(this);



Mime
View raw message