From commits-return-14335-archive-asf-public=cust-asf.ponee.io@archiva.apache.org Fri Mar 30 19:35:53 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id 24895180671 for ; Fri, 30 Mar 2018 19:35:51 +0200 (CEST) Received: (qmail 16863 invoked by uid 500); 30 Mar 2018 17:35:51 -0000 Mailing-List: contact commits-help@archiva.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@archiva.apache.org Delivered-To: mailing list commits@archiva.apache.org Received: (qmail 16844 invoked by uid 99); 30 Mar 2018 17:35:51 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 30 Mar 2018 17:35:51 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 42021EB4FF; Fri, 30 Mar 2018 17:35:49 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: martin_s@apache.org To: commits@archiva.apache.org Date: Fri, 30 Mar 2018 17:35:50 -0000 Message-Id: In-Reply-To: <9f98e9e3610b43efac27b63053d5a7a7@git.apache.org> References: <9f98e9e3610b43efac27b63053d5a7a7@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [2/3] archiva git commit: Cleaning up indexing task Cleaning up indexing task Project: http://git-wip-us.apache.org/repos/asf/archiva/repo Commit: http://git-wip-us.apache.org/repos/asf/archiva/commit/8cb4ef0a Tree: http://git-wip-us.apache.org/repos/asf/archiva/tree/8cb4ef0a Diff: http://git-wip-us.apache.org/repos/asf/archiva/diff/8cb4ef0a Branch: refs/heads/master Commit: 8cb4ef0aeae7b5abb94e1d0f932531424d66a90b Parents: 32acbe2 Author: Martin Stockhammer Authored: Fri Mar 30 17:33:03 2018 +0200 Committer: Martin Stockhammer Committed: Fri Mar 30 17:33:54 2018 +0200 ---------------------------------------------------------------------- .../indexing/ArchivaIndexingTaskExecutor.java | 91 ++++++++------ .../DefaultDownloadRemoteIndexScheduler.java | 4 - .../indexing/DefaultIndexUpdateSideEffect.java | 2 +- .../indexing/DownloadRemoteIndexTask.java | 3 - .../DownloadRemoteIndexTaskRequest.java | 1 - .../indexing/IndexingArchivaTaskScheduler.java | 2 +- .../main/resources/META-INF/spring-context.xml | 13 +- .../ArchivaIndexingTaskExecutorTest.java | 125 +++++++++---------- .../indexing/DownloadRemoteIndexTaskTest.java | 12 +- 9 files changed, 127 insertions(+), 126 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/archiva/blob/8cb4ef0a/archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/java/org/apache/archiva/scheduler/indexing/ArchivaIndexingTaskExecutor.java ---------------------------------------------------------------------- diff --git a/archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/java/org/apache/archiva/scheduler/indexing/ArchivaIndexingTaskExecutor.java b/archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/java/org/apache/archiva/scheduler/indexing/ArchivaIndexingTaskExecutor.java index 03a95be..9a2babf 100644 --- a/archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/java/org/apache/archiva/scheduler/indexing/ArchivaIndexingTaskExecutor.java +++ b/archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/java/org/apache/archiva/scheduler/indexing/ArchivaIndexingTaskExecutor.java @@ -20,8 +20,6 @@ package org.apache.archiva.scheduler.indexing; * under the License. */ -import org.apache.archiva.admin.model.RepositoryAdminException; -import org.apache.archiva.admin.model.managed.ManagedRepositoryAdmin; import org.apache.archiva.indexer.ArchivaIndexingContext; import org.apache.archiva.indexer.UnsupportedBaseContextException; import org.apache.archiva.redback.components.taskqueue.Task; @@ -31,10 +29,15 @@ import org.apache.archiva.repository.ManagedRepository; import org.apache.archiva.repository.features.IndexCreationFeature; import org.apache.maven.index.ArtifactContext; import org.apache.maven.index.ArtifactContextProducer; +import org.apache.maven.index.DefaultScannerListener; import org.apache.maven.index.FlatSearchRequest; import org.apache.maven.index.FlatSearchResponse; +import org.apache.maven.index.Indexer; +import org.apache.maven.index.IndexerEngine; import org.apache.maven.index.MAVEN; -import org.apache.maven.index.NexusIndexer; +import org.apache.maven.index.Scanner; +import org.apache.maven.index.ScanningRequest; +import org.apache.maven.index.ScanningResult; import org.apache.maven.index.context.IndexingContext; import org.apache.maven.index.expr.SourcedSearchExpression; import org.apache.maven.index.packer.IndexPacker; @@ -67,8 +70,13 @@ public class ArchivaIndexingTaskExecutor private ArtifactContextProducer artifactContextProducer; @Inject - private NexusIndexer nexusIndexer; + private Indexer indexer; + @Inject + private Scanner scanner; + + @Inject + IndexerEngine indexerEngine; /** * depending on current {@link Task} you have. @@ -87,30 +95,40 @@ public class ArchivaIndexingTaskExecutor ManagedRepository repository = indexingTask.getRepository( ); ArchivaIndexingContext archivaContext = indexingTask.getContext( ); IndexingContext context = null; - try { - context = archivaContext.getBaseContext(IndexingContext.class); - } catch (UnsupportedBaseContextException e) { - throw new TaskExecutionException("Bad repository type.", e); + try + { + context = archivaContext.getBaseContext( IndexingContext.class ); } - if (!nexusIndexer.getIndexingContexts().containsKey(context.getId())) { - nexusIndexer.addIndexingContext(context); + catch ( UnsupportedBaseContextException e ) + { + throw new TaskExecutionException( "Bad repository type.", e ); } if ( ArtifactIndexingTask.Action.FINISH.equals( indexingTask.getAction( ) ) && indexingTask.isExecuteOnEntireRepo( ) ) { + long start = System.currentTimeMillis( ); try { - long start = System.currentTimeMillis( ); - nexusIndexer.scan( context, null, indexingTask.isOnlyUpdate( ) ); - long end = System.currentTimeMillis( ); - log.info( "indexed maven repository: {}, onlyUpdate: {}, time {} ms", repository.getId( ), - indexingTask.isOnlyUpdate( ), ( end - start ) ); + context.updateTimestamp( ); + DefaultScannerListener listener = new DefaultScannerListener( context, indexerEngine, true, null ); + ScanningRequest request = new ScanningRequest( context, listener ); + ScanningResult result = scanner.scan( request ); + if ( result.hasExceptions( ) ) + { + log.error( "Exceptions occured during index scan of " + context.getId( ) ); + result.getExceptions( ).stream( ).map( e -> e.getMessage( ) ).distinct( ).limit( 5 ).forEach( + s -> log.error( "Message: " + s ) + ); + } } catch ( IOException e ) { - throw new TaskExecutionException( "Error scan repository " + repository, e ); + log.error( "Error during context scan {}: {}", context.getId( ), context.getIndexDirectory( ) ); } + long end = System.currentTimeMillis( ); + log.info( "indexed maven repository: {}, onlyUpdate: {}, time {} ms", repository.getId( ), + indexingTask.isOnlyUpdate( ), ( end - start ) ); log.debug( "Finishing indexing task on repo: {}", repository.getId( ) ); finishIndexingTask( indexingTask, repository, context ); } @@ -125,8 +143,8 @@ public class ArchivaIndexingTaskExecutor ( indexingTask.getResourceFile( ) == null ? "none" : indexingTask.getResourceFile( ) ) ); - archivaContext = repository.getIndexingContext(); - context = archivaContext.getBaseContext(IndexingContext.class); + archivaContext = repository.getIndexingContext( ); + context = archivaContext.getBaseContext( IndexingContext.class ); } catch ( UnsupportedBaseContextException e ) { @@ -167,36 +185,35 @@ public class ArchivaIndexingTaskExecutor //String uinfo = ac.getArtifactInfo().getUinfo(); //TopDocs d = s.search( new TermQuery( new Term( ArtifactInfo.UINFO, uinfo ) ), 1 ); - BooleanQuery q = new BooleanQuery( ); - q.add( nexusIndexer.constructQuery( MAVEN.GROUP_ID, new SourcedSearchExpression( + BooleanQuery.Builder qb = new BooleanQuery.Builder(); + qb.add( indexer.constructQuery( MAVEN.GROUP_ID, new SourcedSearchExpression( ac.getArtifactInfo( ).getGroupId( ) ) ), BooleanClause.Occur.MUST ); - q.add( nexusIndexer.constructQuery( MAVEN.ARTIFACT_ID, new SourcedSearchExpression( + qb.add( indexer.constructQuery( MAVEN.ARTIFACT_ID, new SourcedSearchExpression( ac.getArtifactInfo( ).getArtifactId( ) ) ), BooleanClause.Occur.MUST ); - q.add( nexusIndexer.constructQuery( MAVEN.VERSION, new SourcedSearchExpression( + qb.add( indexer.constructQuery( MAVEN.VERSION, new SourcedSearchExpression( ac.getArtifactInfo( ).getVersion( ) ) ), BooleanClause.Occur.MUST ); if ( ac.getArtifactInfo( ).getClassifier( ) != null ) { - q.add( nexusIndexer.constructQuery( MAVEN.CLASSIFIER, new SourcedSearchExpression( + qb.add( indexer.constructQuery( MAVEN.CLASSIFIER, new SourcedSearchExpression( ac.getArtifactInfo( ).getClassifier( ) ) ), BooleanClause.Occur.MUST ); } if ( ac.getArtifactInfo( ).getPackaging( ) != null ) { - q.add( nexusIndexer.constructQuery( MAVEN.PACKAGING, new SourcedSearchExpression( + qb.add( indexer.constructQuery( MAVEN.PACKAGING, new SourcedSearchExpression( ac.getArtifactInfo( ).getPackaging( ) ) ), BooleanClause.Occur.MUST ); } - FlatSearchRequest flatSearchRequest = new FlatSearchRequest( q, context ); - FlatSearchResponse flatSearchResponse = nexusIndexer.searchFlat( flatSearchRequest ); + FlatSearchRequest flatSearchRequest = new FlatSearchRequest( qb.build(), context ); + FlatSearchResponse flatSearchResponse = indexer.searchFlat( flatSearchRequest ); if ( flatSearchResponse.getResults( ).isEmpty( ) ) { log.debug( "Adding artifact '{}' to index..", ac.getArtifactInfo( ) ); - nexusIndexer.addArtifactToIndex( ac, context ); + indexerEngine.index( context, ac ); } else { log.debug( "Updating artifact '{}' in index..", ac.getArtifactInfo( ) ); // TODO check if update exists !! - nexusIndexer.deleteArtifactFromIndex( ac, context ); - nexusIndexer.addArtifactToIndex( ac, context ); + indexerEngine.update( context, ac ); } context.updateTimestamp( ); @@ -207,7 +224,7 @@ public class ArchivaIndexingTaskExecutor else { log.debug( "Removing artifact '{}' from index..", ac.getArtifactInfo( ) ); - nexusIndexer.deleteArtifactFromIndex( ac, context ); + indexerEngine.remove( context, ac ); } } } @@ -238,26 +255,28 @@ public class ArchivaIndexingTaskExecutor try { - log.debug("Finishing indexing"); + log.debug( "Finishing indexing" ); context.optimize( ); if ( repository.supportsFeature( IndexCreationFeature.class ) ) { IndexCreationFeature icf = repository.getFeature( IndexCreationFeature.class ).get( ); - if ( !icf.isSkipPackedIndexCreation( ) && icf.getLocalPackedIndexPath()!=null) + if ( !icf.isSkipPackedIndexCreation( ) && icf.getLocalPackedIndexPath( ) != null ) { - log.debug("Creating packed index from {} on {}", context.getIndexDirectoryFile(), icf.getLocalPackedIndexPath()); + log.debug( "Creating packed index from {} on {}", context.getIndexDirectoryFile( ), icf.getLocalPackedIndexPath( ) ); IndexPackingRequest request = new IndexPackingRequest( context, // context.acquireIndexSearcher( ).getIndexReader( ), // - icf.getLocalPackedIndexPath().toFile() ); + icf.getLocalPackedIndexPath( ).toFile( ) ); indexPacker.packIndex( request ); context.updateTimestamp( true ); - log.debug( "Index file packed at '{}'.", icf.getLocalPackedIndexPath() ); - } else { + log.debug( "Index file packed at '{}'.", icf.getLocalPackedIndexPath( ) ); + } + else + { log.debug( "skip packed index creation" ); } } http://git-wip-us.apache.org/repos/asf/archiva/blob/8cb4ef0a/archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/java/org/apache/archiva/scheduler/indexing/DefaultDownloadRemoteIndexScheduler.java ---------------------------------------------------------------------- diff --git a/archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/java/org/apache/archiva/scheduler/indexing/DefaultDownloadRemoteIndexScheduler.java b/archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/java/org/apache/archiva/scheduler/indexing/DefaultDownloadRemoteIndexScheduler.java index d5da807..4bf0891 100644 --- a/archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/java/org/apache/archiva/scheduler/indexing/DefaultDownloadRemoteIndexScheduler.java +++ b/archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/java/org/apache/archiva/scheduler/indexing/DefaultDownloadRemoteIndexScheduler.java @@ -21,8 +21,6 @@ package org.apache.archiva.scheduler.indexing; import org.apache.archiva.admin.model.RepositoryAdminException; import org.apache.archiva.admin.model.beans.NetworkProxy; import org.apache.archiva.admin.model.networkproxy.NetworkProxyAdmin; -import org.apache.archiva.common.ArchivaException; -import org.apache.archiva.common.plexusbridge.PlexusSisuBridgeException; import org.apache.archiva.configuration.ArchivaConfiguration; import org.apache.archiva.configuration.ConfigurationEvent; import org.apache.archiva.configuration.ConfigurationListener; @@ -32,7 +30,6 @@ import org.apache.archiva.repository.RepositoryRegistry; import org.apache.archiva.repository.features.RemoteIndexFeature; import org.apache.commons.lang.StringUtils; import org.apache.maven.index.context.IndexingContext; -import org.apache.maven.index.context.UnsupportedExistingLuceneIndexException; import org.apache.maven.index.packer.IndexPacker; import org.apache.maven.index.updater.IndexUpdater; import org.slf4j.Logger; @@ -44,7 +41,6 @@ import org.springframework.stereotype.Service; import javax.annotation.PostConstruct; import javax.inject.Inject; import javax.inject.Named; -import java.io.IOException; import java.util.Date; import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; http://git-wip-us.apache.org/repos/asf/archiva/blob/8cb4ef0a/archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/java/org/apache/archiva/scheduler/indexing/DefaultIndexUpdateSideEffect.java ---------------------------------------------------------------------- diff --git a/archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/java/org/apache/archiva/scheduler/indexing/DefaultIndexUpdateSideEffect.java b/archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/java/org/apache/archiva/scheduler/indexing/DefaultIndexUpdateSideEffect.java index 0dcadd9..e1e2851 100644 --- a/archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/java/org/apache/archiva/scheduler/indexing/DefaultIndexUpdateSideEffect.java +++ b/archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/java/org/apache/archiva/scheduler/indexing/DefaultIndexUpdateSideEffect.java @@ -20,8 +20,8 @@ package org.apache.archiva.scheduler.indexing; */ import org.apache.maven.index.context.IndexingContext; -import org.apache.maven.index_shaded.lucene.store.Directory; import org.apache.maven.index.updater.IndexUpdateSideEffect; +import org.apache.maven.index_shaded.lucene.store.Directory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; http://git-wip-us.apache.org/repos/asf/archiva/blob/8cb4ef0a/archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/java/org/apache/archiva/scheduler/indexing/DownloadRemoteIndexTask.java ---------------------------------------------------------------------- diff --git a/archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/java/org/apache/archiva/scheduler/indexing/DownloadRemoteIndexTask.java b/archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/java/org/apache/archiva/scheduler/indexing/DownloadRemoteIndexTask.java index 63c8b5e..1563e12 100644 --- a/archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/java/org/apache/archiva/scheduler/indexing/DownloadRemoteIndexTask.java +++ b/archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/java/org/apache/archiva/scheduler/indexing/DownloadRemoteIndexTask.java @@ -19,7 +19,6 @@ package org.apache.archiva.scheduler.indexing; */ import org.apache.archiva.admin.model.beans.NetworkProxy; -import org.apache.archiva.admin.model.remote.RemoteRepositoryAdmin; import org.apache.archiva.proxy.common.WagonFactory; import org.apache.archiva.proxy.common.WagonFactoryRequest; import org.apache.archiva.repository.PasswordCredentials; @@ -28,7 +27,6 @@ import org.apache.archiva.repository.RepositoryException; import org.apache.archiva.repository.RepositoryType; import org.apache.archiva.repository.features.RemoteIndexFeature; import org.apache.commons.lang.time.StopWatch; -import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.maven.index.context.IndexingContext; import org.apache.maven.index.updater.IndexUpdateRequest; import org.apache.maven.index.updater.IndexUpdateResult; @@ -54,7 +52,6 @@ import org.slf4j.LoggerFactory; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; -import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; http://git-wip-us.apache.org/repos/asf/archiva/blob/8cb4ef0a/archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/java/org/apache/archiva/scheduler/indexing/DownloadRemoteIndexTaskRequest.java ---------------------------------------------------------------------- diff --git a/archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/java/org/apache/archiva/scheduler/indexing/DownloadRemoteIndexTaskRequest.java b/archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/java/org/apache/archiva/scheduler/indexing/DownloadRemoteIndexTaskRequest.java index 9c5223c..dce7215 100644 --- a/archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/java/org/apache/archiva/scheduler/indexing/DownloadRemoteIndexTaskRequest.java +++ b/archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/java/org/apache/archiva/scheduler/indexing/DownloadRemoteIndexTaskRequest.java @@ -19,7 +19,6 @@ package org.apache.archiva.scheduler.indexing; */ import org.apache.archiva.admin.model.beans.NetworkProxy; -import org.apache.archiva.admin.model.remote.RemoteRepositoryAdmin; import org.apache.archiva.proxy.common.WagonFactory; import org.apache.archiva.repository.RemoteRepository; import org.apache.maven.index.packer.IndexPacker; http://git-wip-us.apache.org/repos/asf/archiva/blob/8cb4ef0a/archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/java/org/apache/archiva/scheduler/indexing/IndexingArchivaTaskScheduler.java ---------------------------------------------------------------------- diff --git a/archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/java/org/apache/archiva/scheduler/indexing/IndexingArchivaTaskScheduler.java b/archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/java/org/apache/archiva/scheduler/indexing/IndexingArchivaTaskScheduler.java index c53410a..3d65810 100644 --- a/archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/java/org/apache/archiva/scheduler/indexing/IndexingArchivaTaskScheduler.java +++ b/archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/java/org/apache/archiva/scheduler/indexing/IndexingArchivaTaskScheduler.java @@ -19,9 +19,9 @@ package org.apache.archiva.scheduler.indexing; * under the License. */ -import org.apache.archiva.scheduler.ArchivaTaskScheduler; import org.apache.archiva.redback.components.taskqueue.TaskQueue; import org.apache.archiva.redback.components.taskqueue.TaskQueueException; +import org.apache.archiva.scheduler.ArchivaTaskScheduler; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; http://git-wip-us.apache.org/repos/asf/archiva/blob/8cb4ef0a/archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/resources/META-INF/spring-context.xml ---------------------------------------------------------------------- diff --git a/archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/resources/META-INF/spring-context.xml b/archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/resources/META-INF/spring-context.xml index aba76b7..674d097 100644 --- a/archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/resources/META-INF/spring-context.xml +++ b/archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/resources/META-INF/spring-context.xml @@ -20,23 +20,14 @@ - - - - - org.apache.maven.index.DefaultNexusIndexer - - + http://git-wip-us.apache.org/repos/asf/archiva/blob/8cb4ef0a/archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/test/java/org/apache/archiva/scheduler/indexing/ArchivaIndexingTaskExecutorTest.java ---------------------------------------------------------------------- diff --git a/archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/test/java/org/apache/archiva/scheduler/indexing/ArchivaIndexingTaskExecutorTest.java b/archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/test/java/org/apache/archiva/scheduler/indexing/ArchivaIndexingTaskExecutorTest.java index 2cc9af6..719a19a 100644 --- a/archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/test/java/org/apache/archiva/scheduler/indexing/ArchivaIndexingTaskExecutorTest.java +++ b/archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/test/java/org/apache/archiva/scheduler/indexing/ArchivaIndexingTaskExecutorTest.java @@ -20,19 +20,19 @@ package org.apache.archiva.scheduler.indexing; */ import junit.framework.TestCase; -import org.apache.archiva.admin.model.managed.ManagedRepositoryAdmin; -import org.apache.archiva.common.utils.PathUtil; import org.apache.archiva.indexer.ArchivaIndexingContext; import org.apache.archiva.indexer.UnsupportedBaseContextException; -import org.apache.archiva.repository.*; -import org.apache.archiva.repository.features.ArtifactCleanupFeature; +import org.apache.archiva.repository.BasicManagedRepository; +import org.apache.archiva.repository.ManagedRepository; +import org.apache.archiva.repository.ReleaseScheme; +import org.apache.archiva.repository.RepositoryRegistry; +import org.apache.archiva.repository.features.IndexCreationFeature; import org.apache.archiva.test.utils.ArchivaSpringJUnit4ClassRunner; import org.apache.maven.index.ArtifactInfo; import org.apache.maven.index.FlatSearchRequest; import org.apache.maven.index.FlatSearchResponse; +import org.apache.maven.index.Indexer; import org.apache.maven.index.MAVEN; -import org.apache.maven.index.NexusIndexer; -import org.apache.maven.index.context.IndexCreator; import org.apache.maven.index.context.IndexingContext; import org.apache.maven.index.expr.SourcedSearchExpression; import org.apache.maven.index.expr.StringSearchExpression; @@ -55,9 +55,6 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.List; -import java.util.Locale; -import java.util.Map; import java.util.Set; /** @@ -71,17 +68,17 @@ public class ArchivaIndexingTaskExecutorTest @Inject private ArchivaIndexingTaskExecutor indexingExecutor; - private BasicManagedRepository repositoryConfig; - - @Inject - private NexusIndexer indexer; - @Inject RepositoryRegistry repositoryRegistry; @Inject private IndexUpdater indexUpdater; + private ManagedRepository repo; + + @Inject + private Indexer indexer; + @Before @Override public void setUp() @@ -90,7 +87,7 @@ public class ArchivaIndexingTaskExecutorTest super.setUp(); Path baseDir = Paths.get(System.getProperty("basedir"), "target/test-classes").toAbsolutePath(); - repositoryConfig = new BasicManagedRepository( "test-repo", "Test Repository", baseDir); + BasicManagedRepository repositoryConfig = new BasicManagedRepository( "test-repo", "Test Repository", baseDir); Path repoLocation = baseDir.resolve("test-repo" ); repositoryConfig.setLocation(repoLocation.toUri() ); repositoryConfig.setLayout( "default" ); @@ -98,6 +95,7 @@ public class ArchivaIndexingTaskExecutorTest repositoryConfig.addActiveReleaseScheme( ReleaseScheme.RELEASE ); repositoryConfig.removeActiveReleaseScheme( ReleaseScheme.SNAPSHOT ); repositoryRegistry.putRepository(repositoryConfig); + repo = repositoryRegistry.getManagedRepository( repositoryConfig.getId() ); } @After @@ -106,10 +104,6 @@ public class ArchivaIndexingTaskExecutorTest throws Exception { - for ( IndexingContext indexingContext : indexer.getIndexingContexts().values() ) - { - indexer.removeIndexingContext( indexingContext, true ); - } repositoryRegistry.destroy(); /* removeIndexingContext with true cleanup files. @@ -126,7 +120,6 @@ public class ArchivaIndexingTaskExecutorTest } protected IndexingContext getIndexingContext() throws UnsupportedBaseContextException { - Repository repo = repositoryRegistry.getRepository(repositoryConfig.getId()); assert repo != null; ArchivaIndexingContext ctx = repo.getIndexingContext(); assert ctx != null; @@ -137,26 +130,29 @@ public class ArchivaIndexingTaskExecutorTest public void testAddArtifactToIndex() throws Exception { - Path basePath = PathUtil.getPathFromUri( repositoryConfig.getLocation() ); + Path basePath = repo.getLocalPath(); Path artifactFile = basePath.resolve( "org/apache/archiva/archiva-index-methods-jar-test/1.0/archiva-index-methods-jar-test-1.0.jar" ); - ManagedRepository repo = repositoryRegistry.getManagedRepository(repositoryConfig.getId()); ArtifactIndexingTask task = - new ArtifactIndexingTask( repositoryConfig, artifactFile, ArtifactIndexingTask.Action.ADD, + new ArtifactIndexingTask( repo, artifactFile, ArtifactIndexingTask.Action.ADD, repo.getIndexingContext()); indexingExecutor.executeTask( task ); - Map ctxs = indexer.getIndexingContexts(); - BooleanQuery q = new BooleanQuery(); - q.add( indexer.constructQuery( MAVEN.GROUP_ID, new StringSearchExpression( "org.apache.archiva" ) ), + task = new ArtifactIndexingTask( repo, null, ArtifactIndexingTask.Action.FINISH, + repo.getIndexingContext() ); + indexingExecutor.executeTask( task ); + + BooleanQuery.Builder queryBuilder = new BooleanQuery.Builder( ); + queryBuilder.add( indexer.constructQuery( MAVEN.GROUP_ID, new StringSearchExpression( "org.apache.archiva" ) ), BooleanClause.Occur.SHOULD ); - q.add( + queryBuilder.add( indexer.constructQuery( MAVEN.ARTIFACT_ID, new StringSearchExpression( "archiva-index-methods-jar-test" ) ), BooleanClause.Occur.SHOULD ); + BooleanQuery q = queryBuilder.build(); - FlatSearchRequest request = new FlatSearchRequest( q ); + FlatSearchRequest request = new FlatSearchRequest( q , getIndexingContext()); FlatSearchResponse response = indexer.searchFlat( request ); assertTrue( Files.exists(basePath.resolve( ".indexer" )) ); @@ -176,29 +172,28 @@ public class ArchivaIndexingTaskExecutorTest public void testUpdateArtifactInIndex() throws Exception { - Path basePath = PathUtil.getPathFromUri( repositoryConfig.getLocation( ) ); + Path basePath = repo.getLocalPath(); Path artifactFile = basePath.resolve( "org/apache/archiva/archiva-index-methods-jar-test/1.0/archiva-index-methods-jar-test-1.0.jar" ); - ManagedRepository repo = repositoryRegistry.getManagedRepository(repositoryConfig.getId()); ArtifactIndexingTask task = - new ArtifactIndexingTask( repositoryConfig, artifactFile, ArtifactIndexingTask.Action.ADD, + new ArtifactIndexingTask( repo, artifactFile, ArtifactIndexingTask.Action.ADD, repo.getIndexingContext() ); indexingExecutor.executeTask( task ); indexingExecutor.executeTask( task ); - BooleanQuery q = new BooleanQuery(); - q.add( indexer.constructQuery( MAVEN.GROUP_ID, new StringSearchExpression( "org.apache.archiva" ) ), + BooleanQuery.Builder qb = new BooleanQuery.Builder(); + qb.add( indexer.constructQuery( MAVEN.GROUP_ID, new StringSearchExpression( "org.apache.archiva" ) ), BooleanClause.Occur.SHOULD ); - q.add( + qb.add( indexer.constructQuery( MAVEN.ARTIFACT_ID, new StringSearchExpression( "archiva-index-methods-jar-test" ) ), BooleanClause.Occur.SHOULD ); IndexingContext ctx = getIndexingContext(); IndexSearcher searcher = ctx.acquireIndexSearcher(); - TopDocs topDocs = searcher.search( q, null, 10 ); + TopDocs topDocs = searcher.search( qb.build(), 10 ); //searcher.close(); ctx.releaseIndexSearcher( searcher ); @@ -214,27 +209,27 @@ public class ArchivaIndexingTaskExecutorTest public void testRemoveArtifactFromIndex() throws Exception { - Path basePath = PathUtil.getPathFromUri( repositoryConfig.getLocation( ) ); + Path basePath = repo.getLocalPath(); Path artifactFile = basePath.resolve( "org/apache/archiva/archiva-index-methods-jar-test/1.0/archiva-index-methods-jar-test-1.0.jar" ); - ManagedRepository repo = repositoryRegistry.getManagedRepository(repositoryConfig.getId()); ArtifactIndexingTask task = - new ArtifactIndexingTask( repositoryConfig, artifactFile, ArtifactIndexingTask.Action.ADD, + new ArtifactIndexingTask( repo, artifactFile, ArtifactIndexingTask.Action.ADD, repo.getIndexingContext() ); // add artifact to index indexingExecutor.executeTask( task ); - BooleanQuery q = new BooleanQuery(); - q.add( indexer.constructQuery( MAVEN.GROUP_ID, new SourcedSearchExpression( "org.apache.archiva" ) ), + BooleanQuery.Builder qb = new BooleanQuery.Builder(); + qb.add( indexer.constructQuery( MAVEN.GROUP_ID, new SourcedSearchExpression( "org.apache.archiva" ) ), BooleanClause.Occur.SHOULD ); //q.add( // indexer.constructQuery( MAVEN.ARTIFACT_ID, new SourcedSearchExpression( "archiva-index-methods-jar-test" ) ), // Occur.SHOULD ); + IndexingContext ctx = repo.getIndexingContext( ).getBaseContext( IndexingContext.class ); FlatSearchRequest flatSearchRequest = - new FlatSearchRequest( q, indexer.getIndexingContexts().get( repositoryConfig.getId() ) ); + new FlatSearchRequest( qb.build(), ctx ); FlatSearchResponse response = indexer.searchFlat( flatSearchRequest ); @@ -245,25 +240,25 @@ public class ArchivaIndexingTaskExecutorTest assertEquals( 1, response.getTotalHitsCount() ); // remove added artifact from index - task = new ArtifactIndexingTask( repositoryConfig, artifactFile, ArtifactIndexingTask.Action.DELETE, + task = new ArtifactIndexingTask( repo, artifactFile, ArtifactIndexingTask.Action.DELETE, repo.getIndexingContext()); indexingExecutor.executeTask( task ); - task = new ArtifactIndexingTask( repositoryConfig, artifactFile, ArtifactIndexingTask.Action.FINISH, + task = new ArtifactIndexingTask( repo, artifactFile, ArtifactIndexingTask.Action.FINISH, repo.getIndexingContext() ); indexingExecutor.executeTask( task ); - q = new BooleanQuery(); - q.add( indexer.constructQuery( MAVEN.GROUP_ID, new SourcedSearchExpression( "org.apache.archiva" ) ), + qb = new BooleanQuery.Builder(); + qb.add( indexer.constructQuery( MAVEN.GROUP_ID, new SourcedSearchExpression( "org.apache.archiva" ) ), BooleanClause.Occur.SHOULD ); - q.add( indexer.constructQuery( MAVEN.ARTIFACT_ID, + qb.add( indexer.constructQuery( MAVEN.ARTIFACT_ID, new SourcedSearchExpression( "archiva-index-methods-jar-test" ) ), BooleanClause.Occur.SHOULD ); assertTrue( Files.exists(basePath.resolve( ".indexer" )) ); assertTrue( Files.exists(basePath.resolve(".index" )) ); - flatSearchRequest = new FlatSearchRequest( q, getIndexingContext() ); + flatSearchRequest = new FlatSearchRequest( qb.build(), getIndexingContext() ); response = indexer.searchFlat( flatSearchRequest ); // artifact should have been removed from the index! @@ -277,14 +272,17 @@ public class ArchivaIndexingTaskExecutorTest throws Exception { - Path basePath = PathUtil.getPathFromUri( repositoryConfig.getLocation()); - Path indexDirectory = basePath.resolve(".index"); + Path basePath = repo.getLocalPath(); + IndexCreationFeature icf = repo.getFeature( IndexCreationFeature.class ).get(); + Path packedIndexDirectory = icf.getLocalPackedIndexPath(); + Path indexerDirectory = icf.getLocalIndexPath(); - Files.list(indexDirectory).filter( path -> path.getFileName().toString().startsWith("nexus-maven-repository-index") ) + Files.list(packedIndexDirectory).filter( path -> path.getFileName().toString().startsWith("nexus-maven-repository-index") ) .forEach( path -> { try { + System.err.println("Deleting "+path); Files.delete( path ); } catch ( IOException e ) @@ -296,48 +294,49 @@ public class ArchivaIndexingTaskExecutorTest Path artifactFile = basePath.resolve( "org/apache/archiva/archiva-index-methods-jar-test/1.0/archiva-index-methods-jar-test-1.0.jar" ); - ManagedRepository repo = repositoryRegistry.getManagedRepository(repositoryConfig.getId()); ArtifactIndexingTask task = - new ArtifactIndexingTask( repositoryConfig, artifactFile, ArtifactIndexingTask.Action.ADD, + new ArtifactIndexingTask( repo, artifactFile, ArtifactIndexingTask.Action.ADD, repo.getIndexingContext() ); task.setExecuteOnEntireRepo( false ); indexingExecutor.executeTask( task ); - task = new ArtifactIndexingTask( repositoryConfig, null, ArtifactIndexingTask.Action.FINISH, + task = new ArtifactIndexingTask( repo, null, ArtifactIndexingTask.Action.FINISH, repo.getIndexingContext() ); task.setExecuteOnEntireRepo( false ); indexingExecutor.executeTask( task ); - assertTrue( Files.exists(indexDirectory) ); + assertTrue( Files.exists(packedIndexDirectory) ); + assertTrue( Files.exists(indexerDirectory) ); // test packed index file creation //no more zip //Assertions.assertThat(new File( indexerDirectory, "nexus-maven-repository-index.zip" )).exists(); - Assertions.assertThat( Files.exists(indexDirectory.resolve("nexus-maven-repository-index.properties" ) )); - Assertions.assertThat( Files.exists(indexDirectory.resolve("nexus-maven-repository-index.gz" ) )); + Assertions.assertThat( Files.exists(packedIndexDirectory.resolve("nexus-maven-repository-index.properties" ) )); + Assertions.assertThat( Files.exists(packedIndexDirectory.resolve("nexus-maven-repository-index.gz" ) )); + assertFalse( Files.exists(packedIndexDirectory.resolve("nexus-maven-repository-index.1.gz" ) )); // unpack .zip index - Path destDir = basePath.resolve( ".index/tmp" ); //unzipIndex( indexerDirectory.getPath(), destDir.getPath() ); - DefaultIndexUpdater.FileFetcher fetcher = new DefaultIndexUpdater.FileFetcher( indexDirectory.toFile() ); + DefaultIndexUpdater.FileFetcher fetcher = new DefaultIndexUpdater.FileFetcher( packedIndexDirectory.toFile() ); IndexUpdateRequest updateRequest = new IndexUpdateRequest( getIndexingContext(), fetcher ); //updateRequest.setLocalIndexCacheDir( indexerDirectory ); indexUpdater.fetchAndUpdateIndex( updateRequest ); - BooleanQuery q = new BooleanQuery(); - q.add( indexer.constructQuery( MAVEN.GROUP_ID, new StringSearchExpression( "org.apache.archiva" ) ), + BooleanQuery.Builder qb = new BooleanQuery.Builder(); + qb.add( indexer.constructQuery( MAVEN.GROUP_ID, new StringSearchExpression( "org.apache.archiva" ) ), BooleanClause.Occur.SHOULD ); - q.add( + qb.add( indexer.constructQuery( MAVEN.ARTIFACT_ID, new StringSearchExpression( "archiva-index-methods-jar-test" ) ), BooleanClause.Occur.SHOULD ); - FlatSearchRequest request = new FlatSearchRequest( q, getIndexingContext() ); + FlatSearchRequest request = new FlatSearchRequest( qb.build(), getIndexingContext() ); FlatSearchResponse response = indexer.searchFlat( request ); + assertEquals( 1, response.getTotalHitsCount() ); Set results = response.getResults(); ArtifactInfo artifactInfo = results.iterator().next(); @@ -345,7 +344,7 @@ public class ArchivaIndexingTaskExecutorTest assertEquals( "archiva-index-methods-jar-test", artifactInfo.getArtifactId() ); assertEquals( "test-repo", artifactInfo.getRepository() ); - assertEquals( 1, response.getTotalHits() ); + } } http://git-wip-us.apache.org/repos/asf/archiva/blob/8cb4ef0a/archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/test/java/org/apache/archiva/scheduler/indexing/DownloadRemoteIndexTaskTest.java ---------------------------------------------------------------------- diff --git a/archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/test/java/org/apache/archiva/scheduler/indexing/DownloadRemoteIndexTaskTest.java b/archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/test/java/org/apache/archiva/scheduler/indexing/DownloadRemoteIndexTaskTest.java index aeeaa67..c4d7d93 100644 --- a/archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/test/java/org/apache/archiva/scheduler/indexing/DownloadRemoteIndexTaskTest.java +++ b/archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/test/java/org/apache/archiva/scheduler/indexing/DownloadRemoteIndexTaskTest.java @@ -23,7 +23,10 @@ import org.apache.archiva.admin.model.remote.RemoteRepositoryAdmin; import org.apache.archiva.common.utils.FileUtils; import org.apache.archiva.repository.RepositoryRegistry; import org.apache.archiva.test.utils.ArchivaSpringJUnit4ClassRunner; -import org.apache.maven.index.*; +import org.apache.maven.index.FlatSearchRequest; +import org.apache.maven.index.FlatSearchResponse; +import org.apache.maven.index.Indexer; +import org.apache.maven.index.MAVEN; import org.apache.maven.index.context.IndexingContext; import org.apache.maven.index.expr.StringSearchExpression; import org.apache.maven.index_shaded.lucene.search.BooleanClause; @@ -81,9 +84,6 @@ public class DownloadRemoteIndexTaskTest @Inject RepositoryRegistry repositoryRegistry; - @Inject - NexusIndexer nexusIndexer; - @Before public void initialize() throws Exception @@ -149,12 +149,12 @@ public class DownloadRemoteIndexTaskTest remoteRepositoryAdmin.deleteRemoteRepository( "test-repo-re", null ); // search - BooleanQuery iQuery = new BooleanQuery(); + BooleanQuery.Builder iQuery = new BooleanQuery.Builder(); iQuery.add( indexer.constructQuery( MAVEN.GROUP_ID, new StringSearchExpression( "commons-logging" ) ), BooleanClause.Occur.SHOULD ); remoteRepositoryAdmin.addRemoteRepository(remoteRepository, null); - FlatSearchRequest rq = new FlatSearchRequest( iQuery ); + FlatSearchRequest rq = new FlatSearchRequest( iQuery.build() ); rq.setContexts( Arrays.asList( repositoryRegistry.getRemoteRepository(remoteRepository.getId()).getIndexingContext().getBaseContext(IndexingContext.class) ) );