lucene-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "Uwe Schindler" <...@thetaphi.de>
Subject RE: svn commit: r1135956 - in /lucene/dev/branches/branch_3x: ./ lucene/ lucene/backwards/ lucene/backwards/src/test-framework/ lucene/backwards/src/test/ solr/ solr/contrib/dataimporthandler/ solr/contrib/dataimporthandler/src/main/java/org/apache/solr/h
Date Thu, 16 Jun 2011 19:35:17 GMT
Shalin,

i had to comment out your test because the finally block does not compile with Java 5 (Solr
3.1), Jenkins is down at the moment, so did not catch earlier.

-----
Uwe Schindler
H.-H.-Meier-Allee 63, D-28213 Bremen
http://www.thetaphi.de
eMail: uwe@thetaphi.de


> -----Original Message-----
> From: shalin@apache.org [mailto:shalin@apache.org]
> Sent: Wednesday, June 15, 2011 10:36 AM
> To: commits@lucene.apache.org
> Subject: svn commit: r1135956 - in /lucene/dev/branches/branch_3x: ./
> lucene/ lucene/backwards/ lucene/backwards/src/test-framework/
> lucene/backwards/src/test/ solr/ solr/contrib/dataimporthandler/
> solr/contrib/dataimporthandler/src/main/java/org/apache/solr/ha...
> 
> Author: shalin
> Date: Wed Jun 15 08:36:06 2011
> New Revision: 1135956
> 
> URL: http://svn.apache.org/viewvc?rev=1135956&view=rev
> Log:
> SOLR-2551 -- Check dataimport.properties for write access (if delta-import is
> supported in DIH configuration) before starting an import
> 
> Modified:
>     lucene/dev/branches/branch_3x/   (props changed)
>     lucene/dev/branches/branch_3x/lucene/   (props changed)
>     lucene/dev/branches/branch_3x/lucene/backwards/   (props changed)
>     lucene/dev/branches/branch_3x/lucene/backwards/src/test/   (props
> changed)
>     lucene/dev/branches/branch_3x/lucene/backwards/src/test-framework/
> (props changed)
>     lucene/dev/branches/branch_3x/solr/   (props changed)
> 
> lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/CHANGES.
> txt
> 
> lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/main/j
> ava/org/apache/solr/handler/dataimport/DataImporter.java
> 
> lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/main/j
> ava/org/apache/solr/handler/dataimport/SolrWriter.java
> 
> lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/test/ja
> va/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta.java
> 
> Modified:
> lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/CHANGES.
> txt
> URL:
> http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/contrib
> /dataimporthandler/CHANGES.txt?rev=1135956&r1=1135955&r2=1135956&vi
> ew=diff
> ==========================================================
> ====================
> ---
> lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/CHANGES.
> txt (original)
> +++
> lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/CHANGES
> +++ .txt Wed Jun 15 08:36:06 2011
> @@ -11,7 +11,8 @@ $Id$
> 
>  ==================  3.3.0-dev ==============
> 
> -(No Changes)
> +* SOLR-2551: Check dataimport.properties for write access (if
> +delta-import is supported
> +  in DIH configuration) before starting an import (C S, shalin)
> 
>  ==================  3.2.0 ==================
> 
> 
> Modified:
> lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/main/j
> ava/org/apache/solr/handler/dataimport/DataImporter.java
> URL:
> http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/contrib
> /dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/Dat
> aImporter.java?rev=1135956&r1=1135955&r2=1135956&view=diff
> ==========================================================
> ====================
> ---
> lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/main/j
> ava/org/apache/solr/handler/dataimport/DataImporter.java (original)
> +++
> lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/mai
> +++ n/java/org/apache/solr/handler/dataimport/DataImporter.java Wed
> Jun
> +++ 15 08:36:06 2011
> @@ -39,6 +39,7 @@ import org.apache.commons.io.IOUtils;
> 
>  import javax.xml.parsers.DocumentBuilder;
>  import javax.xml.parsers.DocumentBuilderFactory;
> +import java.io.File;
>  import java.io.StringReader;
>  import java.text.SimpleDateFormat;
>  import java.util.*;
> @@ -85,6 +86,8 @@ public class DataImporter {
> 
>    private final Map<String , Object> coreScopeSession;
> 
> +  private boolean isDeltaImportSupported = false;
> +
>    /**
>     * Only for testing purposes
>     */
> @@ -113,7 +116,9 @@ public class DataImporter {
>        initEntity(e, fields, false);
>        verifyWithSchema(fields);
>        identifyPk(e);
> -    }
> +      if (e.allAttributes.containsKey(SqlEntityProcessor.DELTA_QUERY))
> +        isDeltaImportSupported = true;
> +    }
>    }
> 
>    private void verifyWithSchema(Map<String, DataConfig.Field> fields) { @@
> -350,6 +355,7 @@ public class DataImporter {
> 
>      try {
>        docBuilder = new DocBuilder(this, writer, requestParams);
> +      checkWritablePersistFile(writer);
>        docBuilder.execute();
>        if (!requestParams.debug)
>          cumulativeStatistics.add(docBuilder.importStatistics);
> @@ -364,6 +370,15 @@ public class DataImporter {
> 
>    }
> 
> +  private void checkWritablePersistFile(SolrWriter writer) {
> +    File persistFile = writer.getPersistFile();
> +    boolean isWritable = persistFile.exists() ? persistFile.canWrite() :
> persistFile.getParentFile().canWrite();
> +    if (isDeltaImportSupported && !isWritable) {
> +      throw new DataImportHandlerException(SEVERE,
> persistFile.getAbsolutePath() +
> +          " is not writable. Delta imports are supported by data config but will
> not work.");
> +    }
> +  }
> +
>    public void doDeltaImport(SolrWriter writer, RequestParams
> requestParams) {
>      LOG.info("Starting Delta Import");
>      setStatus(Status.RUNNING_DELTA_DUMP);
> @@ -371,6 +386,7 @@ public class DataImporter {
>      try {
>        setIndexStartTime(new Date());
>        docBuilder = new DocBuilder(this, writer, requestParams);
> +      checkWritablePersistFile(writer);
>        docBuilder.execute();
>        if (!requestParams.debug)
>          cumulativeStatistics.add(docBuilder.importStatistics);
> 
> Modified:
> lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/main/j
> ava/org/apache/solr/handler/dataimport/SolrWriter.java
> URL:
> http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/contrib
> /dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/Sol
> rWriter.java?rev=1135956&r1=1135955&r2=1135956&view=diff
> ==========================================================
> ====================
> ---
> lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/main/j
> ava/org/apache/solr/handler/dataimport/SolrWriter.java (original)
> +++
> lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/mai
> +++ n/java/org/apache/solr/handler/dataimport/SolrWriter.java Wed Jun 15
> +++ 08:36:06 2011
> @@ -100,13 +100,10 @@ public class SolrWriter {
> 
>      try {
>        props.putAll(p);
> -      String filePath = configDir;
> -      if (configDir != null && !configDir.endsWith(File.separator))
> -        filePath += File.separator;
> -      filePath += persistFilename;
> -      propOutput = new FileOutputStream(filePath);
> +      File persistFile = getPersistFile();
> +      propOutput = new FileOutputStream(persistFile);
>        props.store(propOutput, null);
> -      log.info("Wrote last indexed time to " + persistFilename);
> +      log.info("Wrote last indexed time to " +
> + persistFile.getAbsolutePath());
>      } catch (FileNotFoundException e) {
>        throw new
> DataImportHandlerException(DataImportHandlerException.SEVERE,
>                "Unable to persist Index Start Time", e); @@ -123,6 +120,14 @@
> public class SolrWriter {
>      }
>    }
> 
> +  File getPersistFile() {
> +    String filePath = configDir;
> +    if (configDir != null && !configDir.endsWith(File.separator))
> +      filePath += File.separator;
> +    filePath += persistFilename;
> +    return new File(filePath);
> +  }
> +
>    void finish() {
>      try {
>        processor.finish();
> 
> Modified:
> lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/test/ja
> va/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta.java
> URL:
> http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/contrib
> /dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/Test
> SqlEntityProcessorDelta.java?rev=1135956&r1=1135955&r2=1135956&view=
> diff
> ==========================================================
> ====================
> ---
> lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/test/ja
> va/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta.java
> (original)
> +++
> lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/tes
> +++ t/java/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelt
> +++ a.java Wed Jun 15 08:36:06 2011
> @@ -20,6 +20,8 @@ import org.junit.Before;  import org.junit.BeforeClass;
> import org.junit.Test;
> 
> +import java.io.File;
> +import java.io.FileOutputStream;
>  import java.util.ArrayList;
>  import java.util.Collections;
>  import java.util.List;
> @@ -92,7 +94,37 @@ public class TestSqlEntityProcessorDelta
>    public void testCompositePk_FullImport() throws Exception {
>      add1document();
>    }
> -
> +
> +  @Test
> +  @SuppressWarnings("unchecked")
> +  public void testNonWritablePersistFile() throws Exception {
> +    // See SOLR-2551
> +    String configDir = h.getCore().getResourceLoader().getConfigDir();
> +    String filePath = configDir;
> +    if (configDir != null && !configDir.endsWith(File.separator))
> +      filePath += File.separator;
> +    filePath += "dataimport.properties";
> +    File f = new File(filePath);
> +    // execute the test only if we are able to set file to read only mode
> +    if ((f.exists() || f.createNewFile()) && f.setReadOnly()) {
> +      try {
> +        List parentRow = new ArrayList();
> +        parentRow.add(createMap("id", "1"));
> +        MockDataSource.setIterator(FULLIMPORT_QUERY,
> + parentRow.iterator());
> +
> +        List childRow = new ArrayList();
> +        childRow.add(createMap("desc", "hello"));
> +        MockDataSource.setIterator("select * from y where y.A='1'", childRow
> +            .iterator());
> +
> +        runFullImport(dataConfig_delta);
> +        assertQ(req("id:1"), "//*[@numFound='0']");
> +      } finally {
> +        f.setWritable(true);
> +      }
> +    }
> +  }
> +
>    // WORKS
> 
>    @Test
> 



---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org


Mime
View raw message