lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jd...@apache.org
Subject svn commit: r1414242 - in /lucene/dev/trunk/solr/contrib/dataimporthandler/src: java/org/apache/solr/handler/dataimport/ test/org/apache/solr/handler/dataimport/
Date Tue, 27 Nov 2012 16:12:28 GMT
Author: jdyer
Date: Tue Nov 27 16:12:26 2012
New Revision: 1414242

URL: http://svn.apache.org/viewvc?rev=1414242&view=rev
Log:
SOLR-2141  / SOLR-4047 / SOLR-3842 - fix problems with VariableResolver, better test coverage

Added:
    lucene/dev/trunk/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestVariableResolverEndToEnd.java
  (with props)
Modified:
    lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DateFormatEvaluator.java
    lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java
    lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolver.java
    lucene/dev/trunk/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestBuiltInEvaluators.java

Modified: lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DateFormatEvaluator.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DateFormatEvaluator.java?rev=1414242&r1=1414241&r2=1414242&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DateFormatEvaluator.java
(original)
+++ lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DateFormatEvaluator.java
Tue Nov 27 16:12:26 2012
@@ -67,6 +67,15 @@ public class DateFormatEvaluator extends
       availableLocales.put(locale.toString(), locale);
     }
   }
+  private SimpleDateFormat getDateFormat(String pattern, Locale locale) {
+    DateFormatCacheKey dfck = new DateFormatCacheKey(locale, pattern);
+    SimpleDateFormat sdf = cache.get(dfck);
+    if(sdf == null) {
+      sdf = new SimpleDateFormat(pattern, locale);
+      cache.put(dfck, sdf);
+    }
+    return sdf;
+  }
   
   
   @Override
@@ -81,15 +90,13 @@ public class DateFormatEvaluator extends
       VariableWrapper wrapper = (VariableWrapper) format;
       o = wrapper.resolve();
       format = o.toString();
-    }
+    }    
     Locale locale = Locale.ROOT;
     if(l.size()==3) {
       Object localeObj = l.get(2);
       String localeStr = null;
       if (localeObj  instanceof VariableWrapper) {
-        VariableWrapper wrapper = (VariableWrapper) localeObj;
-        o = wrapper.resolve();
-        localeStr = o.toString();
+        localeStr = ((VariableWrapper) localeObj).resolve().toString();        
       } else {
         localeStr = localeObj.toString();
       }
@@ -97,14 +104,9 @@ public class DateFormatEvaluator extends
       if(locale==null) {
         throw new DataImportHandlerException(SEVERE, "Unsupported locale: " + localeStr);
       }
-    }
+    }    
     String dateFmt = format.toString();
-    DateFormatCacheKey dfck = new DateFormatCacheKey(locale, dateFmt);
-    SimpleDateFormat sdf = cache.get(dfck);
-    if(sdf==null) {
-      sdf = new SimpleDateFormat(dateFmt, locale);
-      cache.put(dfck, sdf);
-    }
+    SimpleDateFormat fmt = getDateFormat(dateFmt, locale);
     Date date = null;
     if (o instanceof VariableWrapper) {
       VariableWrapper variableWrapper = (VariableWrapper) o;
@@ -114,13 +116,7 @@ public class DateFormatEvaluator extends
       } else {
         String s = variableval.toString();
         try {
-          dfck = new DateFormatCacheKey(locale, DEFAULT_DATE_FORMAT);
-          sdf = cache.get(dfck);
-          if(sdf==null) {
-            sdf = new SimpleDateFormat(dfck.dateFormat, dfck.locale);
-            cache.put(dfck, sdf);
-          }
-          date = new SimpleDateFormat(DEFAULT_DATE_FORMAT, locale).parse(s);
+          date = getDateFormat(DEFAULT_DATE_FORMAT, locale).parse(s);
         } catch (ParseException exp) {
           wrapAndThrow(SEVERE, exp, "Invalid expression for date");
         }
@@ -134,7 +130,7 @@ public class DateFormatEvaluator extends
         wrapAndThrow(SEVERE, e, "Invalid expression for date");
       }
     }
-    return sdf.format(date);
+    return fmt.format(date);
   }
   static DateMathParser getDateMathParser(Locale l) {
     return new DateMathParser(TimeZone.getDefault(), l) {

Modified: lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java?rev=1414242&r1=1414241&r2=1414242&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java
(original)
+++ lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java
Tue Nov 27 16:12:26 2012
@@ -68,7 +68,6 @@ public class DocBuilder {
   Map<String, Object> session = new HashMap<String, Object>();
 
   static final ThreadLocal<DocBuilder> INSTANCE = new ThreadLocal<DocBuilder>();
-  //private Map<String, Object> functionsNamespace;
   private Map<String, Object> persistedProperties;
   
   private DIHProperties propWriter;
@@ -640,11 +639,20 @@ public class DocBuilder {
         if (field != null) {
           for (EntityField f : field) {
             String name = f.getName();
+            boolean multiValued = f.isMultiValued();
+            boolean toWrite = f.isToWrite();
             if(f.isDynamicName()){
               name =  vr.replaceTokens(name);
+              SchemaField schemaField = dataImporter.getSchemaField(name);
+              if(schemaField == null) {
+                toWrite = false;
+              } else {
+                multiValued = schemaField.multiValued();
+                toWrite = true;
+              }
             }
-            if (f.isToWrite()) {
-              addFieldToDoc(entry.getValue(), name, f.getBoost(), f.isMultiValued(), doc);
+            if (toWrite) {
+              addFieldToDoc(entry.getValue(), name, f.getBoost(), multiValued, doc);
             }
           }
         }

Modified: lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolver.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolver.java?rev=1414242&r1=1414241&r2=1414242&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolver.java
(original)
+++ lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolver.java
Tue Nov 27 16:12:26 2012
@@ -63,6 +63,7 @@ public class VariableResolver {
   }
   
   public static final String FUNCTIONS_NAMESPACE = "dataimporter.functions.";
+  public static final String FUNCTIONS_NAMESPACE_SHORT = "dih.functions.";
   
   public VariableResolver() {
     rootNamespace = new HashMap<String,Object>();
@@ -95,7 +96,11 @@ public class VariableResolver {
       r = currentLevel.get(nameParts[nameParts.length - 1]);
       if (r == null && name.startsWith(FUNCTIONS_NAMESPACE)
           && name.length() > FUNCTIONS_NAMESPACE.length()) {
-        return resolveEvaluator(name);
+        return resolveEvaluator(FUNCTIONS_NAMESPACE, name);
+      }
+      if (r == null && name.startsWith(FUNCTIONS_NAMESPACE_SHORT)
+          && name.length() > FUNCTIONS_NAMESPACE_SHORT.length()) {
+        return resolveEvaluator(FUNCTIONS_NAMESPACE_SHORT, name);
       }
       if (r == null) {
         r = System.getProperty(name);
@@ -104,12 +109,12 @@ public class VariableResolver {
     return r == null ? "" : r;
   }
   
-  private Object resolveEvaluator(String name) {
+  private Object resolveEvaluator(String namespace, String name) {
     if (evaluators == null) {
       return "";
     }
     Matcher m = EVALUATOR_FORMAT_PATTERN.matcher(name
-        .substring(FUNCTIONS_NAMESPACE.length()));
+        .substring(namespace.length()));
     if (m.find()) {
       String fname = m.group(1);
       Evaluator evaluator = evaluators.get(fname);

Modified: lucene/dev/trunk/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestBuiltInEvaluators.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestBuiltInEvaluators.java?rev=1414242&r1=1414241&r2=1414242&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestBuiltInEvaluators.java
(original)
+++ lucene/dev/trunk/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestBuiltInEvaluators.java
Tue Nov 27 16:12:26 2012
@@ -24,7 +24,7 @@ import java.text.SimpleDateFormat;
 import java.util.*;
 
 /**
- * <p> Test for EvaluatorBag </p>
+ * <p> Test for Evaluators </p>
  *
  *
  * @since solr 1.3

Added: lucene/dev/trunk/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestVariableResolverEndToEnd.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestVariableResolverEndToEnd.java?rev=1414242&view=auto
==============================================================================
--- lucene/dev/trunk/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestVariableResolverEndToEnd.java
(added)
+++ lucene/dev/trunk/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestVariableResolverEndToEnd.java
Tue Nov 27 16:12:26 2012
@@ -0,0 +1,137 @@
+package org.apache.solr.handler.dataimport;
+
+import java.sql.Connection;
+import java.sql.Statement;
+import java.util.Locale;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import junit.framework.Assert;
+
+import org.apache.solr.request.SolrQueryRequest;
+import org.junit.Test;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+public class TestVariableResolverEndToEnd  extends AbstractDIHJdbcTestCase {
+
+  @Test
+  public void test() throws Exception {
+    h.query("/dataimport", generateRequest());
+    SolrQueryRequest req = null;
+    try {
+      req = req("q", "*:*", "wt", "json", "indent", "true");
+      String response = h.query(req);
+      log.debug(response);
+      response = response.replaceAll("\\s","");
+      Assert.assertTrue(response.contains("\"numFound\":1"));
+      Pattern p = Pattern.compile("[\"]second1_s[\"][:][\"](.*?)[\"]");
+      Matcher m = p.matcher(response);
+      Assert.assertTrue(m.find());
+      String yearStr = m.group(1);
+      Assert.assertTrue(response.contains("\"second1_s\":\"" + yearStr + "\""));
+      Assert.assertTrue(response.contains("\"second2_s\":\"" + yearStr + "\""));
+      Assert.assertTrue(response.contains("\"second3_s\":\"" + yearStr + "\""));
+      Assert.assertTrue(response.contains("\"PORK_s\":\"GRILL\""));
+      Assert.assertTrue(response.contains("\"FISH_s\":\"FRY\""));
+      Assert.assertTrue(response.contains("\"BEEF_CUTS_mult_s\":[\"ROUND\",\"SIRLOIN\"]"));
+    } catch(Exception e) {
+      throw e;
+    } finally {
+      req.close();
+    }
+  } 
+  
+  @Override
+  protected String generateConfig() {
+    String thirdLocaleParam = random().nextBoolean() ? "" : (", '" + Locale.getDefault()
+ "'");
+    StringBuilder sb = new StringBuilder();
+    sb.append("<dataConfig> \n");
+    sb.append("<dataSource name=\"hsqldb\" driver=\"org.hsqldb.jdbcDriver\" url=\"jdbc:hsqldb:mem:.\"
/> \n");
+    sb.append("<document name=\"TestEvaluators\"> \n");
+    sb.append("<entity name=\"FIRST\" processor=\"SqlEntityProcessor\" dataSource=\"hsqldb\"
");
+    sb.append(" query=\"" +
+    		"select " +
+    		" 1 as id, " +
+    		" 'SELECT' as SELECT_KEYWORD, " +
+    		" CURRENT_TIMESTAMP as FIRST_TS " +
+    		"from DUAL \" >\n");
+    sb.append("  <field column=\"SELECT_KEYWORD\" name=\"select_keyword_s\" /> \n");
+    sb.append("  <entity name=\"SECOND\" processor=\"SqlEntityProcessor\" dataSource=\"hsqldb\"
transformer=\"TemplateTransformer\" ");
+    sb.append("   query=\"" +
+    		"${dataimporter.functions.encodeUrl(FIRST.SELECT_KEYWORD)} " +
+    		" 1 as SORT, " +
+    		" CURRENT_TIMESTAMP as SECOND_TS, " +
+    		" '${dataimporter.functions.formatDate(FIRST.FIRST_TS, 'yyyy'" + thirdLocaleParam +
")}' as SECOND1_S,  " +
+    		" 'PORK' AS MEAT, " +
+    		" 'GRILL' AS METHOD, " +
+        " 'ROUND' AS CUTS, " +
+        " 'BEEF_CUTS' AS WHATKIND " +
+    		"from DUAL " +
+    		"WHERE 1=${FIRST.ID} " +
+    		"UNION " +    		
+    		"${dataimporter.functions.encodeUrl(FIRST.SELECT_KEYWORD)} " +
+    		" 2 as SORT, " +
+        " CURRENT_TIMESTAMP as SECOND_TS, " +
+        " '${dataimporter.functions.formatDate(FIRST.FIRST_TS, 'yyyy'" + thirdLocaleParam
+ ")}' as SECOND1_S,  " +
+        " 'FISH' AS MEAT, " +
+        " 'FRY' AS METHOD, " +
+        " 'SIRLOIN' AS CUTS, " +
+        " 'BEEF_CUTS' AS WHATKIND " +
+        "from DUAL " +
+        "WHERE 1=${FIRST.ID} " +
+        "ORDER BY SORT \"" +
+    		">\n");
+    sb.append("   <field column=\"SECOND_S\" name=\"second_s\" /> \n");
+    sb.append("   <field column=\"SECOND1_S\" name=\"second1_s\" /> \n");
+    sb.append("   <field column=\"second2_s\" template=\"${dataimporter.functions.formatDate(SECOND.SECOND_TS,
'yyyy'" + thirdLocaleParam + ")}\" /> \n");
+    sb.append("   <field column=\"second3_s\" template=\"${dih.functions.formatDate(SECOND.SECOND_TS,
'yyyy'" + thirdLocaleParam + ")}\" /> \n");
+    sb.append("   <field column=\"METHOD\" name=\"${SECOND.MEAT}_s\"/>\n");
+    sb.append("   <field column=\"CUTS\" name=\"${SECOND.WHATKIND}_mult_s\"/>\n");
+    sb.append("  </entity>\n");
+    sb.append("</entity>\n");
+    sb.append("</document> \n");
+    sb.append("</dataConfig> \n");
+    String config = sb.toString();
+    log.debug(config); 
+    return config;
+  }
+  @Override
+  protected void populateData(Connection conn) throws Exception {
+    Statement s = null;
+    try {
+      s = conn.createStatement();
+      s.executeUpdate("create table dual(dual char(1) not null)");
+      s.executeUpdate("insert into dual values('Y')");
+      conn.commit();
+    } catch (Exception e) {
+      throw e;
+    } finally {
+      try {
+        s.close();
+      } catch (Exception ex) {}
+      try {
+        conn.close();
+      } catch (Exception ex) {}
+    }
+  }
+  @Override
+  protected Database setAllowedDatabases() {
+    return Database.HSQLDB;
+  }  
+}



Mime
View raw message