lucene-solr-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Apache Wiki <wikidi...@apache.org>
Subject [Solr Wiki] Trivial Update of "DataImportHandler" by MarkoBonaci
Date Mon, 04 Oct 2010 07:47:19 GMT
Dear Wiki user,

You have subscribed to a wiki page or wiki category on "Solr Wiki" for change notification.

The "DataImportHandler" page has been changed by MarkoBonaci.
http://wiki.apache.org/solr/DataImportHandler?action=diff&rev1=263&rev2=264

--------------------------------------------------

   * Version 1.2
   * Last revision: 20.09.2010.
   * Author: Marko Bonaci
+ 
-  * Enables scheduling DIH delta and/or full imports
+  * Enables scheduling DIH delta or full imports
+  * Sends HTTP POST request to Solr server
-  * Proofs the concept, needs polishing
+  * Proofs the concept, needs more polishing
-  * Successfully tested on ''Apache Tomcat v6'' (should work on any servlet container)
+  * Successfully tested on ''Apache Tomcat v6'' (should work on any other servlet container)
   * Hasn't been committed to SVN (published only here)
  
  <<BR>>
- <!> Global TODO:
+ <!> TODO:
  
+  * enable user to 
   * add ''cancel'' functionality (to be able to completely disable ''DIHScheduler'' background
thread, without stopping the app/server). Currently, sync can be disabled by setting  ''syncEnabled''
param to anything other than "1" in ''dataimport.properties'', but the background thread still
remains active and reloads the properties file on every run (so that sync can be hot-redeployed)
   * try to use Solr's classes wherever possible
   * add javadoc style comments
@@ -1080, +1083 @@

   * uses [[http://download.oracle.com/javase/6/docs/api/java/util/Properties.html|java.util.Properties]]
to load settings from ''dataimport.properties''
  
  {{{
- package hr.mbo.solr.dataimport;
+ package org.apache.solr.handler.dataimport.scheduler;
  
  import java.io.FileInputStream;
  import java.io.FileNotFoundException;
@@ -1145, +1148 @@

   * Timer is essentially a facility for threads to schedule tasks for future execution in
a background thread.
  
  {{{
- package hr.mbo.solr.scheduler;
+ package org.apache.solr.handler.dataimport.scheduler;
  
  import java.util.Calendar;
  import java.util.Date;
@@ -1224, +1227 @@

  
  <<BR>>
  {{{
+ package org.apache.solr.handler.dataimport.scheduler;
- package hr.mbo.solr.scheduler;
- 
- import hr.mbo.solr.dataimport.SolrDataImportProperties;
  
  import java.io.IOException;
  import java.net.HttpURLConnection;
@@ -1269, +1270 @@

  			logger.info("<index update process> Single core identified in dataimport.properties");
  		}else{
  			singleCore = false;
- 			logger.info("<index update process> Multiple cores identified in dataimport.properties:
" + cores);
+ 			logger.info("<index update process> Multiple cores identified in dataimport.properties.
Sync active for: " + cores);
  		}
  	}
  	
  	private void reloadParams(){
  		p.loadProperties(true);
- 		syncEnabled 	= p.getProperty(SolrDataImportProperties.SYNC_ENABLED);
+ 		syncEnabled = p.getProperty(SolrDataImportProperties.SYNC_ENABLED);
  		cores 		= p.getProperty(SolrDataImportProperties.SYNC_CORES);		
  		server 		= p.getProperty(SolrDataImportProperties.SERVER);
  		port 		= p.getProperty(SolrDataImportProperties.PORT);
  		webapp 		= p.getProperty(SolrDataImportProperties.WEBAPP);
  		params 		= p.getProperty(SolrDataImportProperties.PARAMS);
  		interval	= p.getProperty(SolrDataImportProperties.INTERVAL);
- 		syncCores = cores != null ? cores.split(",") : null;
+ 		syncCores 	= cores != null ? cores.split(",") : null;
  	}
  	
  	private void fixParams(String webAppName){
- 		if(server.isEmpty()) 	server = "localhost";
+ 		if(server == null || server.isEmpty()) 	server = "localhost";
- 		if(port.isEmpty()) 	port = "80";
+ 		if(port == null || port.isEmpty()) 		port = "8080";
- 		if(webapp.isEmpty()) 	webapp = webAppName;
+ 		if(webapp == null || webapp.isEmpty()) 	webapp = webAppName;
- 		if(interval.isEmpty() || getIntervalInt() <= 0) interval = "30";
+ 		if(interval == null || interval.isEmpty() || getIntervalInt() <= 0) interval = "30";
  	}
  	
  	public void run() {
  		try{
  			// check mandatory params
- 			if(server.isEmpty() || webapp.isEmpty() || params.isEmpty()){
+ 			if(server.isEmpty() || webapp.isEmpty() || params == null || params.isEmpty()){
  				logger.warn("<index update process> Insuficient info provided for data import");
  				logger.info("<index update process> Reloading global dataimport.properties");
  				reloadParams();
@@ -1317, +1318 @@

  			}
  		}catch(Exception e){
  			logger.error("Failed to prepare for sendHttpPost", e);
+ 			reloadParams();
  		}
  	}
  	
@@ -1343, +1345 @@

  		
  		try{
  
- 			URL url = new URL(completeUrl);
+ 		    URL url = new URL(completeUrl);
- 			HttpURLConnection conn = (HttpURLConnection)url.openConnection();	
+ 		    HttpURLConnection conn = (HttpURLConnection)url.openConnection();	
  		    
- 			conn.setRequestMethod("POST");
+ 		    conn.setRequestMethod("POST");
- 			conn.setRequestProperty("type", "submit");
+ 		    conn.setRequestProperty("type", "submit");
- 			conn.setDoOutput(true);
+ 		    conn.setDoOutput(true);
  		    
  			// Send HTTP POST		    
- 			conn.connect();
+ 		    conn.connect();
  		    
- 			logger.info(core + "<index update process> Request method\t\t\t" + conn.getRequestMethod());
+ 		    logger.info(core + "<index update process> Request method\t\t\t" + conn.getRequestMethod());
- 			logger.info(core + "<index update process> Succesfully connected to server\t" +
server);		    
+ 		    logger.info(core + "<index update process> Succesfully connected to server\t"
+ server);		    
- 			logger.info(core + "<index update process> Using port\t\t\t" + port);
+ 		    logger.info(core + "<index update process> Using port\t\t\t" + port);
- 			logger.info(core + "<index update process> Application name\t\t\t" + webapp);
+ 		    logger.info(core + "<index update process> Application name\t\t\t" + webapp);
- 			logger.info(core + "<index update process> URL params\t\t\t" + params);
+ 		    logger.info(core + "<index update process> URL params\t\t\t" + params);
- 			logger.info(core + "<index update process> Full URL\t\t\t\t" + conn.getURL());
+ 		    logger.info(core + "<index update process> Full URL\t\t\t\t" + conn.getURL());
- 			logger.info(core + "<index update process> Response message\t\t\t" + conn.getResponseMessage());
+ 		    logger.info(core + "<index update process> Response message\t\t\t" + conn.getResponseMessage());
- 			logger.info(core + "<index update process> Response code\t\t\t" + conn.getResponseCode());
+ 		    logger.info(core + "<index update process> Response code\t\t\t" + conn.getResponseCode());
  		    
- 			//listen for change in properties file if an error occurs 
+ 		    //listen for change in properties file if an error occurs 
- 			if(conn.getResponseCode() != 200){
+ 		    if(conn.getResponseCode() != 200){
- 				reloadParams();
+ 		    	reloadParams();
- 			}
+ 		    }
  		    
- 			conn.disconnect();
+ 		    conn.disconnect();
- 			logger.info(core + "<index update process> Disconnected from server\t\t" + server);
+ 		    logger.info(core + "<index update process> Disconnected from server\t\t" + server);
- 			Date endTime = new Date();
+ 		    Date endTime = new Date();
- 			logger.info(core + "<index update process> Process ended at ................ "
+ df.format(endTime));
+ 		    logger.info(core + "<index update process> Process ended at ................
" + df.format(endTime));
  		}catch(MalformedURLException mue){
  			logger.error("Failed to assemble URL for HTTP POST", mue);
  		}catch(IOException ioe){

Mime
View raw message