oodt-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bfos...@apache.org
Subject svn commit: r1052019 - in /oodt/trunk: CHANGES.txt crawler/src/main/java/org/apache/oodt/cas/crawl/ProductCrawler.java crawler/src/main/java/org/apache/oodt/cas/crawl/status/ crawler/src/main/java/org/apache/oodt/cas/crawl/status/IngestStatus.java
Date Wed, 22 Dec 2010 18:30:29 GMT
Author: bfoster
Date: Wed Dec 22 18:30:29 2010
New Revision: 1052019

URL: http://svn.apache.org/viewvc?rev=1052019&view=rev
Log:

- added product crawl status feature to cas-crawler

---------------------

OODT-90

Added:
    oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/status/
    oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/status/IngestStatus.java  
(with props)
Modified:
    oodt/trunk/CHANGES.txt
    oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/ProductCrawler.java

Modified: oodt/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/oodt/trunk/CHANGES.txt?rev=1052019&r1=1052018&r2=1052019&view=diff
==============================================================================
--- oodt/trunk/CHANGES.txt (original)
+++ oodt/trunk/CHANGES.txt Wed Dec 22 18:30:29 2010
@@ -4,6 +4,8 @@ Apache OODT Change Log
 Release 0.2 (Current Development)
 --------------------------------------------
 
+* OODT-90 (cas-crawler patch) CAS-PGE returns success even if product file(s) failed to ingest
(bfoster)
+
 * OODT-103 modify cas-common's cmd-line parser to throw a special OptionHelpException if
no args 
   are specified so it can be specifically trapped (bfoster)
 

Modified: oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/ProductCrawler.java
URL: http://svn.apache.org/viewvc/oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/ProductCrawler.java?rev=1052019&r1=1052018&r2=1052019&view=diff
==============================================================================
--- oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/ProductCrawler.java (original)
+++ oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/ProductCrawler.java Wed Dec
22 18:30:29 2010
@@ -22,6 +22,7 @@ package org.apache.oodt.cas.crawl;
 import org.apache.oodt.cas.crawl.action.CrawlerAction;
 import org.apache.oodt.cas.crawl.action.CrawlerActionRepo;
 import org.apache.oodt.cas.crawl.config.ProductCrawlerBean;
+import org.apache.oodt.cas.crawl.status.IngestStatus;
 import org.apache.oodt.cas.filemgr.ingest.Ingester;
 import org.apache.oodt.cas.filemgr.ingest.StdIngester;
 import org.apache.oodt.cas.filemgr.metadata.CoreMetKeys;
@@ -31,8 +32,10 @@ import org.apache.oodt.cas.metadata.Meta
 import java.io.File;
 import java.io.FileFilter;
 import java.net.URL;
+import java.util.Collections;
 import java.util.List;
 import java.util.Stack;
+import java.util.Vector;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
@@ -69,7 +72,7 @@ public abstract class ProductCrawler ext
     };
 
     private CrawlerActionRepo actionRepo;
-
+    private List<IngestStatus> ingestStatus;
     private Ingester ingester;
 
     public void crawl() {
@@ -77,6 +80,8 @@ public abstract class ProductCrawler ext
     }
 
     public void crawl(File dirRoot) {
+    	this.ingestStatus = new Vector<IngestStatus>();
+
         // Load actions
         if (this.getApplicationContext() != null)
             (this.actionRepo = new CrawlerActionRepo())
@@ -121,6 +126,10 @@ public abstract class ProductCrawler ext
         }
     }
 
+    public List<IngestStatus> getIngestStatus() {
+    	return Collections.unmodifiableList(this.ingestStatus);
+    }
+    
     private synchronized boolean containsRequiredMetadata(
             Metadata productMetadata) {
         for (int i = 0; i < this.getRequiredMetadata().size(); i++) {
@@ -146,8 +155,10 @@ public abstract class ProductCrawler ext
 					.getAbsoluteFile().getParentFile().getAbsolutePath());
     }
 
-    private void handleFile(File product) {
+    private void handleFile(final File product) {
         LOG.log(Level.INFO, "Handling file " + product);
+    	final IngestStatus.Result ingestResult;
+    	final String message;
         if (this.passesPreconditions(product)) {
             Metadata productMetadata = new Metadata();
             productMetadata.addMetadata(this.getGlobalMetadata().getHashtable());
@@ -158,18 +169,29 @@ public abstract class ProductCrawler ext
             boolean isPreIngestActionsComplete = this.performPreIngestActions(product, productMetadata);
             
             if (this.isSkipIngest()) {
+            	ingestResult = IngestStatus.Result.SKIPPED;
+            	message = "Crawler ingest turned OFF";
                 LOG.log(Level.INFO, "Skipping ingest of product: ["
                     + product.getAbsolutePath() + "]");
             } else {
                 if (isRequiredMetadataPresent
                         && isPreIngestActionsComplete
                         && this.ingest(product, productMetadata)) {
+                	ingestResult = IngestStatus.Result.SUCCESS;
+                	message = "Ingest was successful";
                     LOG.log(Level.INFO, "Successful ingest of product: ["
                             + product.getAbsolutePath() + "]");
                     this
                             .performPostIngestOnSuccessActions(product,
                                     productMetadata);
                 } else {
+                	ingestResult = IngestStatus.Result.FAILURE;
+                	if (!isRequiredMetadataPresent)
+                		message = "Missing required metadata";
+                	else if (!isPreIngestActionsComplete)
+                		message = "PreIngest actions failed to complete";
+                	else
+                		message = "Failed to ingest product";
                     LOG.log(Level.WARNING, "Failed to ingest product: ["
                             + product.getAbsolutePath()
                             + "]: performing postIngestFail actions");
@@ -177,10 +199,23 @@ public abstract class ProductCrawler ext
                 }
             }
         } else {
+        	ingestResult = IngestStatus.Result.PRECONDS_FAILED;
+        	message = "Failed to pass preconditions";
             LOG.log(Level.WARNING,
                     "Failed to pass preconditions for ingest of product: ["
                             + product.getAbsolutePath() + "]");
         }
+        this.ingestStatus.add(new IngestStatus() {
+			public File getProduct() {
+				return product;
+			}
+			public Result getResult() {
+				return ingestResult;
+			}
+			public String getMessage() {
+				return message;
+			}
+        });
     }
 
     private boolean ingest(File product, Metadata productMetdata) {

Added: oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/status/IngestStatus.java
URL: http://svn.apache.org/viewvc/oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/status/IngestStatus.java?rev=1052019&view=auto
==============================================================================
--- oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/status/IngestStatus.java (added)
+++ oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/status/IngestStatus.java Wed
Dec 22 18:30:29 2010
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oodt.cas.crawl.status;
+
+//JDK imports
+import java.io.File;
+
+/**
+ * @author bfoster
+ * @version $Revision$
+ * 
+ * <p>
+ * A Product's Ingest Status
+ * </p>.
+ */
+public interface IngestStatus {
+
+	public static enum Result {
+		SUCCESS, FAILURE, SKIPPED, PRECONDS_FAILED;
+	}
+	
+	public File getProduct();
+	
+	public Result getResult();
+	
+	public String getMessage();
+	
+}

Propchange: oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/status/IngestStatus.java
------------------------------------------------------------------------------
    svn:mime-type = text/plain



Mime
View raw message