Return-Path: X-Original-To: apmail-manifoldcf-commits-archive@www.apache.org Delivered-To: apmail-manifoldcf-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 029EE10712 for ; Fri, 21 Nov 2014 15:43:03 +0000 (UTC) Received: (qmail 2482 invoked by uid 500); 21 Nov 2014 15:43:02 -0000 Delivered-To: apmail-manifoldcf-commits-archive@manifoldcf.apache.org Received: (qmail 2420 invoked by uid 500); 21 Nov 2014 15:43:02 -0000 Mailing-List: contact commits-help@manifoldcf.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@manifoldcf.apache.org Delivered-To: mailing list commits@manifoldcf.apache.org Received: (qmail 2383 invoked by uid 99); 21 Nov 2014 15:43:02 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 21 Nov 2014 15:43:02 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 21 Nov 2014 15:42:38 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id F3FAC2388A6E; Fri, 21 Nov 2014 15:42:36 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1640944 - in /manifoldcf/branches/dev_1x: ./ CHANGES.txt framework/ framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java Date: Fri, 21 Nov 2014 15:42:36 -0000 To: commits@manifoldcf.apache.org From: kwright@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20141121154236.F3FAC2388A6E@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: kwright Date: Fri Nov 21 15:42:36 2014 New Revision: 1640944 URL: http://svn.apache.org/r1640944 Log: Pull up fix for CONNECTORS-1111 from trunk. Modified: manifoldcf/branches/dev_1x/ (props changed) manifoldcf/branches/dev_1x/CHANGES.txt manifoldcf/branches/dev_1x/framework/ (props changed) manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java Propchange: manifoldcf/branches/dev_1x/ ------------------------------------------------------------------------------ Merged /manifoldcf/trunk:r1640941-1640942 Modified: manifoldcf/branches/dev_1x/CHANGES.txt URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/CHANGES.txt?rev=1640944&r1=1640943&r2=1640944&view=diff ============================================================================== --- manifoldcf/branches/dev_1x/CHANGES.txt (original) +++ manifoldcf/branches/dev_1x/CHANGES.txt Fri Nov 21 15:42:36 2014 @@ -3,6 +3,9 @@ $Id$ ======================= 1.8-dev ===================== +CONNECTORS-1111: Component cleanup did not work properly. +(Markus Schuch, Karl Wright) + CONNECTORS-1110: Component documents getting deleted wrongly when additional components added. (Markus Schuch, Karl Wright) Propchange: manifoldcf/branches/dev_1x/framework/ ------------------------------------------------------------------------------ Merged /manifoldcf/trunk/framework:r1640941 Modified: manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java?rev=1640944&r1=1640943&r2=1640944&view=diff ============================================================================== --- manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java (original) +++ manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java Fri Nov 21 15:42:36 2014 @@ -1052,26 +1052,26 @@ public class IncrementalIngester extends // No transactions. Time for the operation may exceed transaction timeout. // Obtain the current URIs of all of these. - DeleteInfo[] uris = getDocumentURIMultiple(outputConnectionName,identifierClasses,identifierHashes); + List uris = getDocumentURIMultiple(outputConnectionName,identifierClasses,identifierHashes); // Grab critical section locks so that we can't attempt to ingest at the same time we are deleting. // (This guarantees that when this operation is complete the database reflects reality.) int validURIcount = 0; - for (int i = 0; i < uris.length; i++) + for (DeleteInfo uri : uris) { - if (uris[i] != null && uris[i].getURI() != null) + if (uri.getURI() != null) validURIcount++; } String[] lockArray = new String[validURIcount]; String[] validURIArray = new String[validURIcount]; String[] validURIHashArray = new String[validURIcount]; validURIcount = 0; - for (int i = 0; i < uris.length; i++) + for (DeleteInfo uri : uris) { - if (uris[i] != null && uris[i].getURI() != null) + if (uri.getURI() != null) { - validURIArray[validURIcount] = uris[i].getURI(); - validURIHashArray[validURIcount] = uris[i].getURIHash(); + validURIArray[validURIcount] = uri.getURI(); + validURIHashArray[validURIcount] = uri.getURIHash(); lockArray[validURIcount] = outputConnectionName+":"+validURIHashArray[validURIcount]; validURIcount++; } @@ -1081,10 +1081,10 @@ public class IncrementalIngester extends try { // Fetch the document URIs for the listed documents - for (int i = 0; i < uris.length; i++) + for (DeleteInfo uri : uris) { - if (uris[i] != null && uris[i].getURI() != null) - removeDocument(connection,uris[i].getURI(),uris[i].getOutputVersion(),activities); + if (uri.getURI() != null) + removeDocument(connection,uri.getURI(),uri.getOutputVersion(),activities); } // Now, get rid of all rows that match the given uris. @@ -1268,26 +1268,26 @@ public class IncrementalIngester extends // No transactions. Time for the operation may exceed transaction timeout. // Obtain the current URIs of all of these. - DeleteInfo[] uris = getDocumentURIMultiple(outputConnectionName,identifierClasses,identifierHashes,componentHash); + List uris = getDocumentURIMultiple(outputConnectionName,identifierClasses,identifierHashes,componentHash); // Grab critical section locks so that we can't attempt to ingest at the same time we are deleting. // (This guarantees that when this operation is complete the database reflects reality.) int validURIcount = 0; - for (int i = 0; i < uris.length; i++) + for (DeleteInfo uri : uris) { - if (uris[i] != null && uris[i].getURI() != null) + if (uri.getURI() != null) validURIcount++; } String[] lockArray = new String[validURIcount]; String[] validURIArray = new String[validURIcount]; String[] validURIHashArray = new String[validURIcount]; validURIcount = 0; - for (int i = 0; i < uris.length; i++) + for (DeleteInfo uri : uris) { - if (uris[i] != null && uris[i].getURI() != null) + if (uri.getURI() != null) { - validURIArray[validURIcount] = uris[i].getURI(); - validURIHashArray[validURIcount] = uris[i].getURIHash(); + validURIArray[validURIcount] = uri.getURI(); + validURIHashArray[validURIcount] = uri.getURIHash(); lockArray[validURIcount] = outputConnectionName+":"+validURIHashArray[validURIcount]; validURIcount++; } @@ -1297,10 +1297,10 @@ public class IncrementalIngester extends try { // Fetch the document URIs for the listed documents - for (int i = 0; i < uris.length; i++) + for (DeleteInfo uri : uris) { - if (uris[i] != null && uris[i].getURI() != null) - removeDocument(connection,uris[i].getURI(),uris[i].getOutputVersion(),activities); + if (uri.getURI() != null) + removeDocument(connection,uri.getURI(),uri.getOutputVersion(),activities); } // Now, get rid of all rows that match the given uris. @@ -1615,37 +1615,29 @@ public class IncrementalIngester extends *@return the array of current document uri's. Null returned for identifiers * that don't exist in the index. */ - protected DeleteInfo[] getDocumentURIMultiple(String outputConnectionName, String[] identifierClasses, String[] identifierHashes) + protected List getDocumentURIMultiple(String outputConnectionName, String[] identifierClasses, String[] identifierHashes) throws ManifoldCFException { - DeleteInfo[] rval = new DeleteInfo[identifierHashes.length]; - Map map = new HashMap(); - for (int i = 0; i < identifierHashes.length; i++) - { - map.put(makeKey(identifierClasses[i],identifierHashes[i]),new Integer(i)); - rval[i] = null; - } - + List rval = new ArrayList(); beginTransaction(); try { List list = new ArrayList(); int maxCount = maxClauseDocumentURIChunk(outputConnectionName); int j = 0; - Iterator iter = map.keySet().iterator(); - while (iter.hasNext()) + for (int i = 0; i < identifierHashes.length; i++) { if (j == maxCount) { - getDocumentURIChunk(rval,map,outputConnectionName,list); + getDocumentURIChunk(rval,outputConnectionName,list); j = 0; list.clear(); } - list.add(iter.next()); + list.add(makeKey(identifierClasses[i],identifierHashes[i])); j++; } if (j > 0) - getDocumentURIChunk(rval,map,outputConnectionName,list); + getDocumentURIChunk(rval,outputConnectionName,list); return rval; } catch (ManifoldCFException e) @@ -1672,37 +1664,29 @@ public class IncrementalIngester extends *@return the array of current document uri's. Null returned for identifiers * that don't exist in the index. */ - protected DeleteInfo[] getDocumentURIMultiple(String outputConnectionName, String[] identifierClasses, String[] identifierHashes, String componentHash) + protected List getDocumentURIMultiple(String outputConnectionName, String[] identifierClasses, String[] identifierHashes, String componentHash) throws ManifoldCFException { - DeleteInfo[] rval = new DeleteInfo[identifierHashes.length]; - Map map = new HashMap(); - for (int i = 0; i < identifierHashes.length; i++) - { - map.put(makeKey(identifierClasses[i],identifierHashes[i]),new Integer(i)); - rval[i] = null; - } - + List rval = new ArrayList(); beginTransaction(); try { List list = new ArrayList(); int maxCount = maxClauseDocumentURIChunk(outputConnectionName,componentHash); int j = 0; - Iterator iter = map.keySet().iterator(); - while (iter.hasNext()) + for (int i = 0; i < identifierHashes.length; i++) { if (j == maxCount) { - getDocumentURIChunk(rval,map,outputConnectionName,list,componentHash); + getDocumentURIChunk(rval,outputConnectionName,list,componentHash); j = 0; list.clear(); } - list.add(iter.next()); + list.add(makeKey(identifierClasses[i],identifierHashes[i])); j++; } if (j > 0) - getDocumentURIChunk(rval,map,outputConnectionName,list,componentHash); + getDocumentURIChunk(rval,outputConnectionName,list,componentHash); return rval; } catch (ManifoldCFException e) @@ -2264,7 +2248,7 @@ public class IncrementalIngester extends *@param clause is the in clause for the query. *@param list are the doc keys for the query. */ - protected void getDocumentURIChunk(DeleteInfo[] rval, Map map, String outputConnectionName, + protected void getDocumentURIChunk(List rval, String outputConnectionName, List list) throws ManifoldCFException { @@ -2280,19 +2264,15 @@ public class IncrementalIngester extends for (int i = 0; i < set.getRowCount(); i++) { IResultRow row = set.getRow(i); - String docHash = row.getValue(docKeyField).toString(); - Integer position = map.get(docHash); - if (position != null) - { - String lastURI = (String)row.getValue(docURIField); - if (lastURI != null && lastURI.length() == 0) - lastURI = null; - String lastURIHash = (String)row.getValue(uriHashField); - if (lastURIHash != null && lastURIHash.length() == 0) - lastURIHash = null; - String lastOutputVersion = (String)row.getValue(lastOutputVersionField); - rval[position.intValue()] = new DeleteInfo(lastURI,lastURIHash,lastOutputVersion); - } + //String docHash = row.getValue(docKeyField).toString(); + String lastURI = (String)row.getValue(docURIField); + if (lastURI != null && lastURI.length() == 0) + lastURI = null; + String lastURIHash = (String)row.getValue(uriHashField); + if (lastURIHash != null && lastURIHash.length() == 0) + lastURIHash = null; + String lastOutputVersion = (String)row.getValue(lastOutputVersionField); + rval.add(new DeleteInfo(lastURI,lastURIHash,lastOutputVersion)); } } @@ -2312,7 +2292,7 @@ public class IncrementalIngester extends *@param list are the doc keys for the query. *@param componentHash is the component hash, if any, for the query. */ - protected void getDocumentURIChunk(DeleteInfo[] rval, Map map, String outputConnectionName, + protected void getDocumentURIChunk(List rval, String outputConnectionName, List list, String componentHash) throws ManifoldCFException { @@ -2329,19 +2309,15 @@ public class IncrementalIngester extends for (int i = 0; i < set.getRowCount(); i++) { IResultRow row = set.getRow(i); - String docHash = row.getValue(docKeyField).toString(); - Integer position = (Integer)map.get(docHash); - if (position != null) - { - String lastURI = (String)row.getValue(docURIField); - if (lastURI != null && lastURI.length() == 0) - lastURI = null; - String lastURIHash = (String)row.getValue(uriHashField); - if (lastURIHash != null && lastURIHash.length() == 0) - lastURIHash = null; - String lastOutputVersion = (String)row.getValue(lastOutputVersionField); - rval[position.intValue()] = new DeleteInfo(lastURI,lastURIHash,lastOutputVersion); - } + //String docHash = row.getValue(docKeyField).toString(); + String lastURI = (String)row.getValue(docURIField); + if (lastURI != null && lastURI.length() == 0) + lastURI = null; + String lastURIHash = (String)row.getValue(uriHashField); + if (lastURIHash != null && lastURIHash.length() == 0) + lastURIHash = null; + String lastOutputVersion = (String)row.getValue(lastOutputVersionField); + rval.add(new DeleteInfo(lastURI,lastURIHash,lastOutputVersion)); } }