Return-Path: Delivered-To: apmail-archiva-commits-archive@www.apache.org Received: (qmail 24416 invoked from network); 2 Dec 2009 02:21:31 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.3) by minotaur.apache.org with SMTP; 2 Dec 2009 02:21:31 -0000 Received: (qmail 69195 invoked by uid 500); 2 Dec 2009 02:21:31 -0000 Delivered-To: apmail-archiva-commits-archive@archiva.apache.org Received: (qmail 69133 invoked by uid 500); 2 Dec 2009 02:21:31 -0000 Mailing-List: contact commits-help@archiva.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@archiva.apache.org Delivered-To: mailing list commits@archiva.apache.org Received: (qmail 69124 invoked by uid 99); 2 Dec 2009 02:21:31 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 02 Dec 2009 02:21:31 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 02 Dec 2009 02:21:28 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 8F74F23889B8; Wed, 2 Dec 2009 02:21:06 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r886035 - in /archiva/branches/MRM-1025/archiva-modules: archiva-base/archiva-consumers/archiva-database-consumers/src/main/java/org/apache/maven/archiva/consumers/database/ archiva-reporting/archiva-artifact-reports/ archiva-reporting/arch... Date: Wed, 02 Dec 2009 02:21:06 -0000 To: commits@archiva.apache.org From: brett@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20091202022106.8F74F23889B8@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: brett Date: Wed Dec 2 02:21:05 2009 New Revision: 886035 URL: http://svn.apache.org/viewvc?rev=886035&view=rev Log: [MRM-1025] change duplicate artifact reporting to be a repository scanned consumer to facilitate removing the database unprocessed consumers Removed: archiva/branches/MRM-1025/archiva-modules/archiva-reporting/archiva-project-reports/src/main/java/org/apache/maven/archiva/reporting/project/ProjectDependenciesConsumer.java Modified: archiva/branches/MRM-1025/archiva-modules/archiva-base/archiva-consumers/archiva-database-consumers/src/main/java/org/apache/maven/archiva/consumers/database/ProjectModelToDatabaseConsumer.java archiva/branches/MRM-1025/archiva-modules/archiva-reporting/archiva-artifact-reports/pom.xml archiva/branches/MRM-1025/archiva-modules/archiva-reporting/archiva-artifact-reports/src/main/java/org/apache/maven/archiva/reporting/artifact/DuplicateArtifactsConsumer.java archiva/branches/MRM-1025/archiva-modules/archiva-reporting/archiva-artifact-reports/src/test/java/org/apache/maven/archiva/reporting/artifact/DuplicateArtifactReportTest.java Modified: archiva/branches/MRM-1025/archiva-modules/archiva-base/archiva-consumers/archiva-database-consumers/src/main/java/org/apache/maven/archiva/consumers/database/ProjectModelToDatabaseConsumer.java URL: http://svn.apache.org/viewvc/archiva/branches/MRM-1025/archiva-modules/archiva-base/archiva-consumers/archiva-database-consumers/src/main/java/org/apache/maven/archiva/consumers/database/ProjectModelToDatabaseConsumer.java?rev=886035&r1=886034&r2=886035&view=diff ============================================================================== --- archiva/branches/MRM-1025/archiva-modules/archiva-base/archiva-consumers/archiva-database-consumers/src/main/java/org/apache/maven/archiva/consumers/database/ProjectModelToDatabaseConsumer.java (original) +++ archiva/branches/MRM-1025/archiva-modules/archiva-base/archiva-consumers/archiva-database-consumers/src/main/java/org/apache/maven/archiva/consumers/database/ProjectModelToDatabaseConsumer.java Wed Dec 2 02:21:05 2009 @@ -9,7 +9,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an @@ -34,10 +34,7 @@ import org.apache.maven.archiva.model.ArchivaArtifact; import org.apache.maven.archiva.model.ArchivaModelCloner; import org.apache.maven.archiva.model.ArchivaProjectModel; -import org.apache.maven.archiva.model.CiManagement; -import org.apache.maven.archiva.model.IssueManagement; import org.apache.maven.archiva.model.Keys; -import org.apache.maven.archiva.model.Organization; import org.apache.maven.archiva.model.RepositoryProblem; import org.apache.maven.archiva.reporting.artifact.CorruptArtifactReport; import org.apache.maven.archiva.repository.ManagedRepositoryContent; Modified: archiva/branches/MRM-1025/archiva-modules/archiva-reporting/archiva-artifact-reports/pom.xml URL: http://svn.apache.org/viewvc/archiva/branches/MRM-1025/archiva-modules/archiva-reporting/archiva-artifact-reports/pom.xml?rev=886035&r1=886034&r2=886035&view=diff ============================================================================== --- archiva/branches/MRM-1025/archiva-modules/archiva-reporting/archiva-artifact-reports/pom.xml (original) +++ archiva/branches/MRM-1025/archiva-modules/archiva-reporting/archiva-artifact-reports/pom.xml Wed Dec 2 02:21:05 2009 @@ -42,6 +42,10 @@ org.codehaus.plexus + plexus-digest + + + org.codehaus.plexus plexus-utils Modified: archiva/branches/MRM-1025/archiva-modules/archiva-reporting/archiva-artifact-reports/src/main/java/org/apache/maven/archiva/reporting/artifact/DuplicateArtifactsConsumer.java URL: http://svn.apache.org/viewvc/archiva/branches/MRM-1025/archiva-modules/archiva-reporting/archiva-artifact-reports/src/main/java/org/apache/maven/archiva/reporting/artifact/DuplicateArtifactsConsumer.java?rev=886035&r1=886034&r2=886035&view=diff ============================================================================== --- archiva/branches/MRM-1025/archiva-modules/archiva-reporting/archiva-artifact-reports/src/main/java/org/apache/maven/archiva/reporting/artifact/DuplicateArtifactsConsumer.java (original) +++ archiva/branches/MRM-1025/archiva-modules/archiva-reporting/archiva-artifact-reports/src/main/java/org/apache/maven/archiva/reporting/artifact/DuplicateArtifactsConsumer.java Wed Dec 2 02:21:05 2009 @@ -19,25 +19,32 @@ * under the License. */ +import java.io.File; import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; import java.util.List; import org.apache.commons.collections.CollectionUtils; import org.apache.maven.archiva.configuration.ArchivaConfiguration; import org.apache.maven.archiva.configuration.ConfigurationNames; import org.apache.maven.archiva.configuration.FileTypes; +import org.apache.maven.archiva.configuration.ManagedRepositoryConfiguration; import org.apache.maven.archiva.consumers.AbstractMonitoredConsumer; import org.apache.maven.archiva.consumers.ConsumerException; +import org.apache.maven.archiva.consumers.KnownRepositoryContentConsumer; import org.apache.maven.archiva.database.ArchivaDAO; import org.apache.maven.archiva.database.ArchivaDatabaseException; import org.apache.maven.archiva.database.ObjectNotFoundException; import org.apache.maven.archiva.database.constraints.ArtifactsByChecksumConstraint; -import org.apache.maven.archiva.database.updater.ArchivaArtifactConsumer; import org.apache.maven.archiva.model.ArchivaArtifact; import org.apache.maven.archiva.model.RepositoryProblem; import org.apache.maven.archiva.repository.ManagedRepositoryContent; import org.apache.maven.archiva.repository.RepositoryContentFactory; import org.apache.maven.archiva.repository.RepositoryException; +import org.apache.maven.archiva.repository.layout.LayoutException; +import org.codehaus.plexus.digest.Digester; +import org.codehaus.plexus.digest.DigesterException; import org.codehaus.plexus.personality.plexus.lifecycle.phase.Initializable; import org.codehaus.plexus.personality.plexus.lifecycle.phase.InitializationException; import org.codehaus.plexus.registry.Registry; @@ -48,17 +55,19 @@ /** * Search the database of known SHA1 Checksums for potential duplicate artifacts. * + * TODO: no need for this to be a scanner - we can just query the database / content repository to get a full list + * * @version $Id$ - * - * @plexus.component role="org.apache.maven.archiva.database.updater.ArchivaArtifactConsumer" - * role-hint="duplicate-artifacts" + * @plexus.component role="org.apache.maven.archiva.consumers.KnownRepositoryContentConsumer" + * role-hint="duplicate-artifacts" + * instantiation-strategy="per-lookup" */ public class DuplicateArtifactsConsumer extends AbstractMonitoredConsumer - implements ArchivaArtifactConsumer, RegistryListener, Initializable + implements KnownRepositoryContentConsumer, RegistryListener, Initializable { private Logger log = LoggerFactory.getLogger( DuplicateArtifactsConsumer.class ); - + /** * @plexus.configuration default-value="duplicate-artifacts" */ @@ -89,9 +98,19 @@ */ private RepositoryContentFactory repositoryFactory; - // TODO: why is this not used? If it should be, what about excludes? private List includes = new ArrayList(); + private File repositoryDir; + + /** + * @plexus.requirement role-hint="sha1" + */ + private Digester digestSha1; + + private String repoId; + + private ManagedRepositoryContent repository; + public String getId() { return id; @@ -107,40 +126,62 @@ return false; } - public void beginScan() + public List getIncludes() { - /* do nothing */ + return includes; } - public void completeScan() + public List getExcludes() { - /* do nothing */ + return Collections.emptyList(); } - public List getIncludedTypes() + public void beginScan( ManagedRepositoryConfiguration repo, Date whenGathered ) + throws ConsumerException { - return null; + try + { + repoId = repo.getId(); + repository = repositoryFactory.getManagedRepositoryContent( repoId ); + this.repositoryDir = new File( repository.getRepoRoot() ); + } + catch ( RepositoryException e ) + { + throw new ConsumerException( e.getMessage(), e ); + } } - public void processArchivaArtifact( ArchivaArtifact artifact ) + public void processFile( String path ) throws ConsumerException { - String checksumSha1 = artifact.getModel().getChecksumSHA1(); + File artifactFile = new File( this.repositoryDir, path ); - List results = null; + // TODO: would be quicker to somehow make sure it ran after the update database consumer, or as a part of that + // perhaps could use an artifact context that is retained for all consumers? First in can set the SHA-1 + String checksumSha1; try { - results = dao.getArtifactDAO().queryArtifacts( new ArtifactsByChecksumConstraint( - checksumSha1, ArtifactsByChecksumConstraint.SHA1 ) ); + checksumSha1 = digestSha1.calc( artifactFile ); + } + catch ( DigesterException e ) + { + throw new ConsumerException( e.getMessage(), e ); + } + + List results; + try + { + results = dao.getArtifactDAO().queryArtifacts( + new ArtifactsByChecksumConstraint( checksumSha1, ArtifactsByChecksumConstraint.SHA1 ) ); } catch ( ObjectNotFoundException e ) { - log.debug( "No duplicates for artifact: " + artifact ); + log.debug( "No duplicates for artifact: " + path + " (repository " + repoId + ")" ); return; } catch ( ArchivaDatabaseException e ) { - log.warn( "Unable to query DB for potential duplicates with : " + artifact ); + log.warn( "Unable to query DB for potential duplicates with: " + path + " (repository " + repoId + "): " + e.getMessage(), e ); return; } @@ -149,10 +190,20 @@ if ( results.size() <= 1 ) { // No duplicates detected. - log.debug( "Found no duplicate artifact results on: " + artifact ); + log.debug( "Found no duplicate artifact results on: " + path + " (repository " + repoId + ")" ); return; } + ArchivaArtifact artifact; + try + { + artifact = new ArchivaArtifact( repository.toArtifactReference( path ), repoId ); + } + catch ( LayoutException e ) + { + log.warn( "Unable to report problem for path: " + path ); + return; + } for ( ArchivaArtifact dupArtifact : results ) { if ( dupArtifact.equals( artifact ) ) @@ -163,7 +214,7 @@ RepositoryProblem problem = new RepositoryProblem(); problem.setRepositoryId( dupArtifact.getModel().getRepositoryId() ); - problem.setPath( toPath( dupArtifact ) ); + problem.setPath( path ); problem.setGroupId( artifact.getGroupId() ); problem.setArtifactId( artifact.getArtifactId() ); problem.setVersion( artifact.getVersion() ); @@ -186,19 +237,9 @@ } } - private String toPath( ArchivaArtifact artifact ) + public void completeScan() { - try - { - String repoId = artifact.getModel().getRepositoryId(); - ManagedRepositoryContent repo = repositoryFactory.getManagedRepositoryContent( repoId ); - return repo.toPath( artifact ); - } - catch ( RepositoryException e ) - { - log.warn( "Unable to calculate path for artifact: " + artifact ); - return ""; - } + // nothing to do } public void afterConfigurationChange( Registry registry, String propertyName, Object propertyValue ) Modified: archiva/branches/MRM-1025/archiva-modules/archiva-reporting/archiva-artifact-reports/src/test/java/org/apache/maven/archiva/reporting/artifact/DuplicateArtifactReportTest.java URL: http://svn.apache.org/viewvc/archiva/branches/MRM-1025/archiva-modules/archiva-reporting/archiva-artifact-reports/src/test/java/org/apache/maven/archiva/reporting/artifact/DuplicateArtifactReportTest.java?rev=886035&r1=886034&r2=886035&view=diff ============================================================================== --- archiva/branches/MRM-1025/archiva-modules/archiva-reporting/archiva-artifact-reports/src/test/java/org/apache/maven/archiva/reporting/artifact/DuplicateArtifactReportTest.java (original) +++ archiva/branches/MRM-1025/archiva-modules/archiva-reporting/archiva-artifact-reports/src/test/java/org/apache/maven/archiva/reporting/artifact/DuplicateArtifactReportTest.java Wed Dec 2 02:21:05 2009 @@ -20,17 +20,20 @@ */ import java.io.File; +import java.io.IOException; import java.util.Date; import java.util.List; import org.apache.commons.io.FileUtils; import org.apache.maven.archiva.configuration.ArchivaConfiguration; import org.apache.maven.archiva.configuration.ManagedRepositoryConfiguration; +import org.apache.maven.archiva.consumers.KnownRepositoryContentConsumer; import org.apache.maven.archiva.database.ArtifactDAO; -import org.apache.maven.archiva.database.updater.ArchivaArtifactConsumer; import org.apache.maven.archiva.model.ArchivaArtifact; import org.apache.maven.archiva.model.RepositoryProblem; import org.apache.maven.archiva.reporting.DynamicReportSource; +import org.apache.maven.archiva.repository.ManagedRepositoryContent; +import org.apache.maven.archiva.repository.RepositoryContentFactory; /** * DuplicateArtifactReportTest @@ -42,14 +45,18 @@ { private static final String TESTABLE_REPO = "testable"; - private static final String HASH3 = "f3f653289f3217c65324830ab3415bc92feddefa"; + private static final String HASH3 = "94ca33031e37aa3f3b67e5b921c729f08a6bba75"; - private static final String HASH2 = "a49810ad3eba8651677ab57cd40a0f76fdef9538"; + private static final String HASH2 = "43f7aa390f1a0265fc2de7010133951c0718a67e"; - private static final String HASH1 = "232f01b24b1617c46a3d4b0ab3415bc9237dcdec"; + private static final String HASH1 = "8107759ababcbfa34bcb02bc4309caf6354982ab"; private ArtifactDAO artifactDao; + private ManagedRepositoryConfiguration repoConfig; + + private ManagedRepositoryContent content; + @Override protected void setUp() throws Exception @@ -60,19 +67,41 @@ ArchivaConfiguration config = (ArchivaConfiguration) lookup( ArchivaConfiguration.class.getName(), "default" ); - ManagedRepositoryConfiguration repoConfig = new ManagedRepositoryConfiguration(); + repoConfig = new ManagedRepositoryConfiguration(); repoConfig.setId( TESTABLE_REPO ); repoConfig.setLayout( "default" ); File testRepoDir = new File( getBasedir(), "target/test-repository" ); FileUtils.forceMkdir( testRepoDir ); repoConfig.setLocation( testRepoDir.getAbsolutePath() ); config.getConfiguration().addManagedRepository( repoConfig ); + + RepositoryContentFactory factory = (RepositoryContentFactory) lookup( RepositoryContentFactory.class ); + content = factory.getManagedRepositoryContent( TESTABLE_REPO ); + + createArtifactFile( testRepoDir, "test-one", "1.0", "value1" ); + createArtifactFile( testRepoDir, "test-one", "1.1", "value1" ); + createArtifactFile( testRepoDir, "test-one", "1.2", "value1" ); + createArtifactFile( testRepoDir, "test-two", "1.0", "value1" ); + createArtifactFile( testRepoDir, "test-two", "2.0", "value3" ); + createArtifactFile( testRepoDir, "test-two", "2.1", "value2" ); + createArtifactFile( testRepoDir, "test-two", "3.0", "value2" ); + } + + private void createArtifactFile( File testRepoDir, String artifactId, String version, String value ) + throws IOException + { + File file = new File( testRepoDir, + "org/apache/maven/archiva/test/" + artifactId + "/" + version + "/" + artifactId + "-" + + version + ".jar" ); + file.getParentFile().mkdirs(); + FileUtils.writeStringToFile( file, value ); } public ArchivaArtifact createArtifact( String artifactId, String version ) { ArchivaArtifact artifact = - artifactDao.createArtifact( "org.apache.maven.archiva.test", artifactId, version, "", "jar", TESTABLE_REPO ); + artifactDao.createArtifact( "org.apache.maven.archiva.test", artifactId, version, "", "jar", + TESTABLE_REPO ); artifact.getModel().setLastModified( new Date() ); return artifact; } @@ -142,14 +171,15 @@ throws Exception { List artifacts = dao.getArtifactDAO().queryArtifacts( null ); - ArchivaArtifactConsumer consumer = - (ArchivaArtifactConsumer) lookup( ArchivaArtifactConsumer.class.getName(), "duplicate-artifacts" ); - consumer.beginScan(); + KnownRepositoryContentConsumer consumer = + (KnownRepositoryContentConsumer) lookup( KnownRepositoryContentConsumer.class.getName(), + "duplicate-artifacts" ); + consumer.beginScan( repoConfig, new Date() ); try { for ( ArchivaArtifact artifact : artifacts ) { - consumer.processArchivaArtifact( artifact ); + consumer.processFile( content.toPath( artifact ) ); } } finally