diff --git a/java/maven.indexer/src/org/netbeans/modules/maven/indexer/ClassDependencyIndexCreator.java b/java/maven.indexer/src/org/netbeans/modules/maven/indexer/ClassDependencyIndexCreator.java index 3fc430635fb4..70304a285182 100644 --- a/java/maven.indexer/src/org/netbeans/modules/maven/indexer/ClassDependencyIndexCreator.java +++ b/java/maven.indexer/src/org/netbeans/modules/maven/indexer/ClassDependencyIndexCreator.java @@ -168,7 +168,7 @@ private static boolean isArchiveFile(File jar) throws IOException { static void search(String className, Indexer indexer, Collection contexts, List results) throws IOException { String searchString = crc32base32(className.replace('.', '/')); Query refClassQuery = indexer.constructQuery(FLD_NB_DEPENDENCY_CLASS.getOntology(), new StringSearchExpression(searchString)); - TopScoreDocCollector collector = TopScoreDocCollector.create(NexusRepositoryIndexerImpl.MAX_RESULT_COUNT, Integer.MAX_VALUE); + TopScoreDocCollector collector = TopScoreDocCollector.create(NexusRepositoryIndexManager.MAX_RESULT_COUNT, Integer.MAX_VALUE); for (IndexingContext context : contexts) { IndexSearcher searcher = context.acquireIndexSearcher(); try { @@ -185,7 +185,7 @@ static void search(String className, Indexer indexer, Collection version = NexusRepositoryIndexerImpl.convertToNBVersionInfo(List.of(ai)); + List version = NexusRepositoryQueries.convertToNBVersionInfo(List.of(ai)); if (!version.isEmpty()) { results.add(new ClassUsage(version.get(0), refClasses)); } diff --git a/java/maven.indexer/src/org/netbeans/modules/maven/indexer/NexusRepositoryIndexerImpl.java b/java/maven.indexer/src/org/netbeans/modules/maven/indexer/NexusRepositoryIndexManager.java similarity index 51% rename from java/maven.indexer/src/org/netbeans/modules/maven/indexer/NexusRepositoryIndexerImpl.java rename to java/maven.indexer/src/org/netbeans/modules/maven/indexer/NexusRepositoryIndexManager.java index bffe61e34114..57d832e89bbe 100644 --- a/java/maven.indexer/src/org/netbeans/modules/maven/indexer/NexusRepositoryIndexerImpl.java +++ b/java/maven.indexer/src/org/netbeans/modules/maven/indexer/NexusRepositoryIndexManager.java @@ -41,13 +41,10 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.logging.Level; import java.util.logging.Logger; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import java.util.stream.Stream; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.Term; -import org.apache.lucene.queryparser.classic.QueryParser; import org.codehaus.plexus.PlexusConstants; import org.apache.lucene.search.*; import org.apache.lucene.store.AlreadyClosedException; @@ -66,17 +63,10 @@ import org.apache.maven.index.creator.MavenArchetypeArtifactInfoIndexCreator; import org.apache.maven.index.creator.MavenPluginArtifactInfoIndexCreator; import org.apache.maven.index.creator.MinimalArtifactInfoIndexCreator; -import org.apache.maven.index.expr.StringSearchExpression; import org.apache.maven.index.updater.IndexUpdateRequest; import org.apache.maven.index.updater.IndexUpdateResult; import org.apache.maven.index.updater.IndexUpdater; import org.apache.maven.index.updater.ResourceFetcher; -import org.apache.maven.search.api.SearchRequest; -import org.apache.maven.search.api.request.FieldQuery; -import org.apache.maven.search.api.request.Paging; -import org.apache.maven.search.api.transport.Java11HttpClientTransport; -import org.apache.maven.search.backend.smo.SmoSearchBackend; -import org.apache.maven.search.backend.smo.SmoSearchBackendFactory; import org.apache.maven.settings.Proxy; import org.apache.maven.settings.Server; import org.apache.maven.settings.crypto.DefaultSettingsDecryptionRequest; @@ -102,16 +92,9 @@ import org.codehaus.plexus.component.repository.ComponentDescriptor; import org.codehaus.plexus.component.repository.ComponentRequirement; import org.codehaus.plexus.component.repository.exception.ComponentLookupException; -import org.netbeans.api.annotations.common.CheckForNull; -import org.netbeans.api.annotations.common.NullAllowed; import org.netbeans.modules.maven.embedder.EmbedderFactory; -import org.netbeans.modules.maven.indexer.api.NBArtifactInfo; -import org.netbeans.modules.maven.indexer.api.NBGroupInfo; -import org.netbeans.modules.maven.indexer.api.NBVersionInfo; -import org.netbeans.modules.maven.indexer.api.QueryField; import org.netbeans.modules.maven.indexer.api.RepositoryInfo; import org.netbeans.modules.maven.indexer.api.RepositoryPreferences; -import org.netbeans.modules.maven.indexer.api.RepositoryQueries; import org.netbeans.modules.maven.indexer.spi.ArchetypeQueries; import org.netbeans.modules.maven.indexer.spi.BaseQueries; import org.netbeans.modules.maven.indexer.spi.ChecksumQueries; @@ -121,7 +104,6 @@ import org.netbeans.modules.maven.indexer.spi.DependencyInfoQueries; import org.netbeans.modules.maven.indexer.spi.GenericFindQuery; import org.netbeans.modules.maven.indexer.spi.impl.RepositoryIndexerImplementation; -import org.netbeans.modules.maven.indexer.spi.ResultImplementation; import org.netbeans.modules.maven.indexer.spi.impl.IndexingNotificationProvider; import org.openide.modules.Places; import org.openide.util.BaseUtilities; @@ -145,11 +127,9 @@ @ServiceProvider(service=RepositoryIndexerImplementation.class), @ServiceProvider(service=RepositoryIndexQueryProvider.class, position = Integer.MAX_VALUE) }) -public class NexusRepositoryIndexerImpl implements RepositoryIndexerImplementation, RepositoryIndexQueryProvider, - BaseQueries, ChecksumQueries, ArchetypeQueries, DependencyInfoQueries, - ClassesQuery, ClassUsageQuery, GenericFindQuery, ContextLoadedQuery { +public final class NexusRepositoryIndexManager implements RepositoryIndexerImplementation, RepositoryIndexQueryProvider { - private static final Logger LOGGER = Logger.getLogger(NexusRepositoryIndexerImpl.class.getName()); + private static final Logger LOGGER = Logger.getLogger(NexusRepositoryIndexManager.class.getName()); private static final String GROUP_CACHE_ALL_PREFIX = "nb-groupcache-all-v1"; // NOI18N private static final String GROUP_CACHE_ALL_SUFFIX = "txt"; // NOI18N @@ -182,77 +162,79 @@ public class NexusRepositoryIndexerImpl implements RepositoryIndexerImplementati */ private static final RequestProcessor RP_REMOTE = new RequestProcessor("maven-remote-indexing"); - private SmoSearchBackend smo = null; + private final NexusRepositoryQueries queries; + + static final int MAX_RESULT_COUNT = 1024; + static final int NO_CAP_RESULT_COUNT = AbstractSearchRequest.UNDEFINED; + + @SuppressWarnings("this-escape") + public NexusRepositoryIndexManager() { + this.queries = new NexusRepositoryQueries(this); + } @Override public boolean handlesRepository(RepositoryInfo repo) { // should always come as last when looked up // handles all remote repos - return true; + return true; } @Override public BaseQueries getBaseQueries() { - return this; + return queries; } @Override public ChecksumQueries getChecksumQueries() { - return this; + return queries; } @Override public ArchetypeQueries getArchetypeQueries() { - return this; + return queries; } @Override public DependencyInfoQueries getDependencyInfoQueries() { - return this; + return queries; } @Override public ClassesQuery getClassesQuery() { - return this; + return queries; } @Override public ClassUsageQuery getClassUsageQuery() { - return this; + return queries; } @Override public GenericFindQuery getGenericFindQuery() { - return this; + return queries; } @Override public ContextLoadedQuery getContextLoadedQuery() { - return this; + return queries; } - private static Mutex getRepoMutex(RepositoryInfo repo) { + static Mutex getRepoMutex(RepositoryInfo repo) { return getRepoMutex(repo.getId()); } - private static Mutex getRepoMutex(String repoId) { + static Mutex getRepoMutex(String repoId) { synchronized (repoMutexMap) { return repoMutexMap.computeIfAbsent(repoId, k -> new Mutex()); } } - - static final int MAX_RESULT_COUNT = 1024; - static final int NO_CAP_RESULT_COUNT = AbstractSearchRequest.UNDEFINED; - - public NexusRepositoryIndexerImpl() { - } private void initIndexer () { if (!inited) { try { ContainerConfiguration config = new DefaultContainerConfiguration(); //#154755 - start - ClassLoader indexerLoader = NexusRepositoryIndexerImpl.class.getClassLoader(); + ClassLoader indexerLoader = NexusRepositoryIndexManager.class.getClassLoader(); ClassWorld classWorld = new ClassWorld(); ClassRealm plexusRealm = classWorld.newRealm("plexus.core", EmbedderFactory.class.getClassLoader()); //NOI18N plexusRealm.importFrom(indexerLoader, "META-INF/sisu"); //NOI18N @@ -315,15 +297,15 @@ private IndexingContext addIndexingContextForced( } indexingContexts.put(context.getId(), context); return context; - } + } public void removeIndexingContext(IndexingContext context, boolean deleteFiles) throws IOException { if (indexingContexts.remove(context.getId()) != null ) { indexer.closeIndexingContext( context, deleteFiles ); } - } + } - private boolean loadIndexingContext(final RepositoryInfo info) throws IOException { + boolean loadIndexingContext(final RepositoryInfo info) throws IOException { LOAD: { assert getRepoMutex(info).isWriteAccess(); initIndexer(); @@ -346,7 +328,6 @@ private boolean loadIndexingContext(final RepositoryInfo info) throws IOExceptio break LOAD; // XXX does it suffice to just return here, or is code after block needed? } } - LOGGER.log(Level.FINE, "Loading Context: {0}", info.getId()); List creators; if (info.isLocal()) { @@ -360,13 +341,19 @@ private boolean loadIndexingContext(final RepositoryInfo info) throws IOExceptio ); } else { creators = List.of( - info.getId().equals(getSMO().getRepositoryId()) + info.getId().equals(queries.getSMO().getRepositoryId()) ? new MinimalArtifactInfoRemoteIndexCreator() : new MinimalArtifactInfoIndexCreator(), new NotifyingIndexCreator() ); } + + if (info.isRemoteDownloadable() && !indexExists(getIndexDirectory(info))) { + tryMoveRemoteIndexFromOldCache(info); + } + try { + LOGGER.log(Level.FINE, "Loading Context: {0}", info.getId()); addIndexingContextForced(info, creators); LOGGER.log(Level.FINE, "using index creators: {0}", creators); } catch (IOException | IllegalArgumentException ex) { // IAE thrown by lucene on index version incompatibilites @@ -405,53 +392,59 @@ private boolean loadIndexingContext(final RepositoryInfo info) throws IOExceptio } } - private @CheckForNull IteratorSearchResponse repeatedPagedSearch(Query q, IndexingContext context, int count) throws IOException { - return repeatedPagedSearch(q, List.of(context), count); - } + /// Moves the index from the previous cache folder to the current folder if: + /// - both cache folders share the same parent + /// - the cache folder is from an older release + /// - the index itself is still somewhat up to date + /// - current NB instance is not a dev build + private void tryMoveRemoteIndexFromOldCache(RepositoryInfo info) { - private @CheckForNull IteratorSearchResponse repeatedPagedSearch(Query q, List contexts, int count) throws IOException { - IteratorSearchRequest isr = new IteratorSearchRequest(q, contexts, new NoJavadocSourceFilter()); - if (count > 0) { - isr.setCount(count); + String buildnumber = System.getProperty("netbeans.buildnumber"); + if (buildnumber == null) { + return; // tests } - - int MAX_MAX_CLAUSE = 1<<11; // conservative maximum for too general queries, like "c:*class*" - - if (q instanceof BooleanQuery booleanQuery) { - List list = booleanQuery.clauses(); - if (list.size() == 1) { - Query q1 = list.get(0).getQuery(); - if (q1 instanceof PrefixQuery pq && "u".equals(pq.getPrefix().field())) { - // increase for queries like "+u:org.netbeans.modules|*" to succeed - MAX_MAX_CLAUSE = 1<<16; - } else if (q1 instanceof TermQuery tq && "p".equals(tq.getTerm().field())) { - // +p:nbm also produces several thousand hits - MAX_MAX_CLAUSE = 1<<16; - } - } + int ourVersion; + try { + String debugRelease = System.getProperty("maven.indexing.diag.release"); + ourVersion = debugRelease != null + ? Integer.parseInt(debugRelease) + : Integer.parseInt(buildnumber.split("-")[0]); + } catch (NumberFormatException ignore) { + return; } - int oldMax = IndexSearcher.getMaxClauseCount(); - try { - int max = oldMax; - while (true) { - IteratorSearchResponse response; - try { - IndexSearcher.setMaxClauseCount(max); - response = searcher.searchIteratorPaged(isr, contexts); - LOGGER.log(Level.FINE, "passed on {0} clauses processing {1} with {2} hits", new Object[] {max, q, response.getTotalHitsCount()}); - return response; - } catch (IndexSearcher.TooManyClauses exc) { - LOGGER.log(Level.FINE, "TooManyClauses on {0} clauses processing {1}", new Object[] {max, q}); - max *= 2; - if (max > MAX_MAX_CLAUSE) { - LOGGER.log(Level.WARNING, "Encountered more than {0} clauses processing {1}", new Object[] {MAX_MAX_CLAUSE, q}); - return null; + Path cacheParent = Places.getCacheDirectory().toPath().getParent(); + if (cacheParent != null) { + try (Stream caches = Files.list(cacheParent)) { + record CacheFolder(Path path, int version) {} + + Optional oldCache = caches + .filter(path -> Files.exists(path.resolve(".lastUsedVersion"))) + .map(path -> { + try { + int version = Integer.parseInt(Files.readString(path.resolve(".lastUsedVersion"))); + return new CacheFolder(path, version); + } catch (IOException | NumberFormatException ex) { + return null; + } + }) + .filter(Objects::nonNull) + .filter(cache -> cache.version < ourVersion) + .max((c1, c2) -> c1.version - c2.version); + + if (oldCache.isPresent()) { + Path oldIndex = getIndexDirectory(oldCache.get().path, info); + if (indexExists(oldIndex)) { + Instant lastUse = Files.getLastModifiedTime(oldIndex.resolve("timestamp")).toInstant(); + if (Instant.now().minus(30, ChronoUnit.DAYS).isBefore(lastUse)) { + LOGGER.log(Level.INFO, "moving index from old cache [{0}]", oldIndex); + Files.move(oldIndex, getIndexDirectory(info)); + } } } + } catch (Exception ex) { + LOGGER.log(Level.WARNING, "index import failed: {0}", ex.getMessage()); } - } finally { - IndexSearcher.setMaxClauseCount(oldMax); } } @@ -483,9 +476,8 @@ private void indexLoadedRepo(final RepositoryInfo repo, boolean updateLocal) thr indexingMutexes.add(mutex); } boolean fetchFailed = false; + boolean fullUpdate = false; long t = System.currentTimeMillis(); - IndexUpdateResult fetchUpdateResult = null; - RemoteIndexTransferListener listener = null; try { IndexingContext indexingContext = getIndexingContexts().get(repo.getId()); if (indexingContext == null) { @@ -494,8 +486,7 @@ private void indexLoadedRepo(final RepositoryInfo repo, boolean updateLocal) thr } if (repo.isRemoteDownloadable()) { LOGGER.log(Level.FINE, "Indexing Remote Repository: {0}", repo.getId()); - listener = new RemoteIndexTransferListener(repo); - try { + try (RemoteIndexTransferListener listener = new RemoteIndexTransferListener(repo)) { String protocol = URI.create(indexingContext.getIndexUpdateUrl()).getScheme(); SettingsDecryptionResult settings = embedder.lookup(SettingsDecrypter.class).decrypt(new DefaultSettingsDecryptionRequest(EmbedderFactory.getOnlineEmbedder().getSettings())); AuthenticationInfo wagonAuth = null; @@ -564,16 +555,40 @@ private void indexLoadedRepo(final RepositoryInfo repo, boolean updateLocal) thr } try { removeGroupCache(repo); - fetchUpdateResult = remoteIndexUpdater.fetchAndUpdateIndex(iur); + IndexUpdateResult result = remoteIndexUpdater.fetchAndUpdateIndex(iur); + fullUpdate = result.isFullUpdate(); storeGroupCache(repo, indexingContext); // register indexed repo in services view - if (fetchUpdateResult.isFullUpdate() && fetchUpdateResult.isSuccessful()) { + if (result.isFullUpdate() && result.isSuccessful()) { RepositoryPreferences.getInstance().addOrModifyRepositoryInfo(repo); } } catch (IOException | AlreadyClosedException | IllegalArgumentException ex) { // AlreadyClosedException can happen in low storage situations when lucene is trying to handle IOEs // IllegalArgumentException signals remote archive format problems fetchFailed = true; + + Path tmpFolder = Path.of(System.getProperty("java.io.tmpdir")); + Path cacheFolder = getIndexDirectory(); + + long freeTmpSpace = getFreeSpaceInMB(tmpFolder); + long freeCacheSpace = getFreeSpaceInMB(cacheFolder); + + if (isNoSpaceLeftOnDevice(ex) || freeCacheSpace < 1000 || freeTmpSpace < 1000) { + + long downloaded = listener.getUnits() * 1024; + LOGGER.log(Level.INFO, "Downloaded maven index file has size {0} (zipped). The usable space in [cache]:{1} is {2} MB and in [tmp]:{3} is {4} MB.", + new Object[] {downloaded, cacheFolder, freeCacheSpace, tmpFolder, freeTmpSpace}); + LOGGER.log(Level.WARNING, "Download/Extraction failed due to low storage, indexing is now disabled.", ex); + + // disable indexing and tell user about it + RepositoryPreferences.setIndexRepositories(false); + + IndexingNotificationProvider np = Lookup.getDefault().lookup(IndexingNotificationProvider.class); + if(np != null) { + np.notifyError(Bundle.MSG_NoSpace(repo.getName(), "[cache]:"+cacheFolder.toString(), freeCacheSpace, "[tmp]:"+tmpFolder.toString(), freeTmpSpace)); + } + unloadIndexingContext(repo.getId()); + } throw new IOException("Failed to load maven-index for: " + indexingContext.getRepositoryUrl(), ex); } catch (RuntimeException ex) { // thread pools, like the one used in maven-indexer's IndexDataReader, may suppress cancellation exceptions @@ -597,10 +612,8 @@ private void indexLoadedRepo(final RepositoryInfo repo, boolean updateLocal) thr LOGGER.log(Level.WARNING, "cleanup failed"); } } - } finally { - listener.close(); } - } else { + } else if (repo.isLocal()) { LOGGER.log(Level.FINE, "Indexing Local Repository: {0}", repo.getId()); if (!indexingContext.getRepository().exists()) { //#210743 @@ -621,28 +634,6 @@ private void indexLoadedRepo(final RepositoryInfo repo, boolean updateLocal) thr if(e.getCause() instanceof ResourceDoesNotExistException) { fireChange(repo, () -> repo.fireNoIndex()); } - Path tmpFolder = Path.of(System.getProperty("java.io.tmpdir")); - Path cacheFolder = getIndexDirectory(); - - long freeTmpSpace = getFreeSpaceInMB(tmpFolder); - long freeCacheSpace = getFreeSpaceInMB(cacheFolder); - - if (isNoSpaceLeftOnDevice(e) || freeCacheSpace < 1000 || freeTmpSpace < 1000) { - - long downloaded = listener != null ? listener.getUnits() * 1024 : -1; - LOGGER.log(Level.INFO, "Downloaded maven index file has size {0} (zipped). The usable space in [cache]:{1} is {2} MB and in [tmp]:{3} is {4} MB.", - new Object[] {downloaded, cacheFolder, freeCacheSpace, tmpFolder, freeTmpSpace}); - LOGGER.log(Level.WARNING, "Download/Extraction failed due to low storage, indexing is now disabled.", e); - - // disable indexing and tell user about it - RepositoryPreferences.setIndexRepositories(false); - - IndexingNotificationProvider np = Lookup.getDefault().lookup(IndexingNotificationProvider.class); - if(np != null) { - np.notifyError(Bundle.MSG_NoSpace(repo.getName(), "[cache]:"+cacheFolder.toString(), freeCacheSpace, "[tmp]:"+tmpFolder.toString(), freeTmpSpace)); - } - unloadIndexingContext(repo.getId()); - } throw e; } catch (Cancellation x) { pauseRemoteRepoIndexing(120); // pause a while @@ -651,12 +642,12 @@ private void indexLoadedRepo(final RepositoryInfo repo, boolean updateLocal) thr throw new IOException("could not find protocol handler for " + repo.getRepositoryUrl(), x); } finally { String kind; - if (fetchUpdateResult != null) { - kind = fetchUpdateResult.isFullUpdate() ? "download, create" : "incremental download, update"; + if (repo.isRemoteDownloadable()) { + kind = fullUpdate ? "download, create" : "incremental download, update"; } else { - kind = "scan"; + kind = repo.isLocal() ? "scan" : "none"; } - LOGGER.log(Level.INFO, "Indexing [{0}] of {1} took {2}s.", new Object[]{kind, repo.getId(), String.format("%.2f", (System.currentTimeMillis() - t)/1000.0f)}); + LOGGER.log(Level.INFO, "Indexing [{0}] of {1} took {2}s.", new Object[]{kind, repo.getId(), "%.2f".formatted((System.currentTimeMillis() - t)/1000.0f)}); synchronized (indexingMutexes) { indexingMutexes.remove(mutex); } @@ -678,7 +669,7 @@ private static boolean isNoSpaceLeftOnDevice(Throwable ex) { Throwable[] suppressed = ex.getSuppressed(); return (msg != null && msg.contains("No space left on device")) || (cause != null && isNoSpaceLeftOnDevice(cause)) - || (suppressed.length > 0 && Stream.of(suppressed).anyMatch(NexusRepositoryIndexerImpl::isNoSpaceLeftOnDevice)); + || (suppressed.length > 0 && Stream.of(suppressed).anyMatch(NexusRepositoryIndexManager::isNoSpaceLeftOnDevice)); } private static boolean isCancellation(Throwable ex) { @@ -690,7 +681,7 @@ private static boolean isDiag() { } //spawn the indexing into a separate thread.. - private boolean spawnIndexLoadedRepo(final RepositoryInfo repo) { + boolean spawnIndexLoadedRepo(final RepositoryInfo repo) { if (shouldSkipIndexRequest(repo)) { return false; @@ -716,7 +707,7 @@ private boolean spawnIndexLoadedRepo(final RepositoryInfo repo) { }); }); return true; - } + } @Override public void indexRepo(final RepositoryInfo repo) { @@ -839,7 +830,7 @@ private void scanLocalRepo(IndexingContext context, String fromPath, ArtifactSca } removeDir(tmpDir); } - } + } @Override public void updateIndexWithArtifacts(final RepositoryInfo repo, final Collection artifacts) { @@ -849,10 +840,10 @@ public void updateIndexWithArtifacts(final RepositoryInfo repo, final Collection final ArtifactRepository repository = EmbedderFactory.getProjectEmbedder().getLocalRepository(); try { getRepoMutex(repo).writeAccess((Mutex.ExceptionAction) () -> { - boolean index = loadIndexingContext(repo); - if (index) { + boolean index = loadIndexingContext(repo); + if (index) { //do not bother indexing - return null; + return null; } IndexingContext indexingContext = indexingContexts.get(repo.getId()); if (indexingContext == null) { @@ -862,7 +853,7 @@ public void updateIndexWithArtifacts(final RepositoryInfo repo, final Collection if (!indexingContext.getRepository().exists()) { //#210743 - LOGGER.log(Level.FINE, "Local repository at {0} doesn't exist, no update.", indexingContext.getRepository()); + LOGGER.log(Level.FINE, "Local repository at {0} doesn't exist, no update.", indexingContext.getRepository()); return null; } Set artifactContexts = new HashSet<>(); @@ -885,7 +876,7 @@ public void updateIndexWithArtifacts(final RepositoryInfo repo, final Collection BooleanQuery bq = new BooleanQuery.Builder() .add(new BooleanClause(new PrefixQuery(new Term(ArtifactInfo.UINFO, id)), BooleanClause.Occur.MUST)) .build(); - try (IteratorSearchResponse response = repeatedPagedSearch(bq, indexingContext, MAX_RESULT_COUNT)) { + try (IteratorSearchResponse response = queries.repeatedPagedSearch(bq, indexingContext, MAX_RESULT_COUNT)) { add = response == null || response.getTotalHitsCount() == 0; } } @@ -919,13 +910,13 @@ public void updateIndexWithArtifacts(final RepositoryInfo repo, final Collection @Override public void deleteArtifactFromIndex(final RepositoryInfo repo, final Artifact artifact) { if (!RepositoryPreferences.isIndexRepositories()) { - return; + return; } final ArtifactRepository repository = EmbedderFactory.getProjectEmbedder().getLocalRepository(); try { getRepoMutex(repo).writeAccess((Mutex.ExceptionAction) () -> { boolean index = loadIndexingContext(repo); - if (index) { + if (index) { return null; //do not bother indexing } IndexingContext indexingContext = indexingContexts.get(repo.getId()); @@ -935,7 +926,7 @@ public void deleteArtifactFromIndex(final RepositoryInfo repo, final Artifact ar } if (!indexingContext.getRepository().exists()) { //#210743 - LOGGER.log(Level.FINE, "Local repository at {0} doesn't exist, no update.", indexingContext.getRepository()); + LOGGER.log(Level.FINE, "Local repository at {0} doesn't exist, no update.", indexingContext.getRepository()); return null; } @@ -977,688 +968,13 @@ private void fireChange(final RepositoryInfo repo, Runnable r) { r.run(); } - @Override - public ResultImplementation getGroups(List repos) { - return filterGroupIds("", repos); - } - - private static boolean isIndexing(Mutex mutex) { + static boolean isIndexing(Mutex mutex) { synchronized (indexingMutexes) { return indexingMutexes.contains(mutex); } } - - private interface RepoAction { - void run(RepositoryInfo repo, IndexingContext context) throws IOException; - } - - private void iterate(List repos, final RepoAction action, final RepoAction actionSkip, final boolean skipUnIndexed) { - if (repos == null) { - repos = RepositoryPreferences.getInstance().getRepositoryInfos(); - } - for (final RepositoryInfo repo : repos) { - Mutex mutex = getRepoMutex(repo); - if (skipUnIndexed && isIndexing(mutex)) { - try { - actionSkip.run(repo, null); - } catch (IOException ex) { - LOGGER.log(Level.FINER, "could not skip " + repo.getId(), ex); - } - } else { - mutex.writeAccess((Mutex.Action) () -> { - try { - boolean index = loadIndexingContext(repo); - if (skipUnIndexed && index) { - if (!RepositoryPreferences.isIndexRepositories()) { - return null; - } - boolean spawned = spawnIndexLoadedRepo(repo); - if (spawned) { - actionSkip.run(repo, null); - } - return null; - } - IndexingContext context = getIndexingContexts().get(repo.getId()); - if (context == null) { - if (skipUnIndexed) { - actionSkip.run(repo, null); - } - return null; - } - action.run(repo, context); - } catch (IOException x) { - LOGGER.log(Level.INFO, "could not process " + repo.getId(), x); - } - return null; - }); - } - } - } - - private ResultImplementation filterGroupIds(final String prefix, final List repos) { - ResultImpl result = new ResultImpl<>( - (ResultImpl res) -> filterGroupIds(prefix, res, res.getSkipped(), false) - ); - return filterGroupIds(prefix, result, repos, true); - } - - private ResultImplementation filterGroupIds(final String prefix, final ResultImpl result, - final List repos, final boolean skipUnIndexed) { - final Set groups = new TreeSet<>(result.getResults()); - final SkippedAction skipAction = new SkippedAction(result); - - iterate(repos, (RepositoryInfo repo, IndexingContext context) -> { - Set all= context.getAllGroups(); - if (!all.isEmpty()) { - if (prefix.isEmpty()) { - groups.addAll(all); - } else { - for (String gr : all) { - if (gr.startsWith(prefix)) { - groups.add(gr); - } - } - } - } - }, skipAction, skipUnIndexed); - - result.setResults(groups); - return result; - } - - @Override - public ResultImplementation getGAVsForPackaging(final String packaging, List repos) { - ResultImpl result = new ResultImpl<>((ResultImpl result1) -> { - getGAVsForPackaging(packaging, result1, result1.getSkipped(), false); - }); - return getGAVsForPackaging(packaging,result, repos, true); - } - - private ResultImplementation getGAVsForPackaging(final String packaging, final ResultImpl result, List repos, final boolean skipUnIndexed) { - final List infos = new ArrayList<>(result.getResults()); - final SkippedAction skipAction = new SkippedAction(result); - BooleanQuery bq = new BooleanQuery.Builder() - .add(new BooleanClause(new TermQuery(new Term(ArtifactInfo.PACKAGING, packaging)), BooleanClause.Occur.MUST)) - .build(); - iterate(repos, (RepositoryInfo repo, IndexingContext context) -> { - IteratorSearchResponse response = repeatedPagedSearch(bq, context, NO_CAP_RESULT_COUNT); - if (response != null) { - try { - for (ArtifactInfo ai : response) { - String gav = ai.getGroupId() + ":" + ai.getArtifactId() + ":" + ai.getVersion(); - if (!infos.contains(gav)) { - infos.add(gav); - } - } - } finally { - result.addReturnedResultCount(response.getTotalProcessedArtifactInfoCount()); - result.addTotalResultCount(response.getTotalHitsCount()); - response.close(); - } - } - }, skipAction, skipUnIndexed); - result.setResults(infos); - return result; - } - - @Override - public ResultImplementation getRecords(final String groupId, final String artifactId, final String version, List repos) { - ResultImpl result = new ResultImpl<>((ResultImpl result1) -> { - getRecords(groupId, artifactId, version, result1, result1.getSkipped(), false); - }); - return getRecords(groupId, artifactId, version, result, repos, true); - } - - private ResultImplementation getRecords(final String groupId, final String artifactId, final String version, final ResultImpl result, - List repos, final boolean skipUnIndexed) { - final List infos = new ArrayList<>(result.getResults()); - final SkippedAction skipAction = new SkippedAction(result); - String id = groupId + ArtifactInfo.FS + artifactId + ArtifactInfo.FS + version + ArtifactInfo.FS; - BooleanQuery bq = new BooleanQuery.Builder() - .add(new BooleanClause(new PrefixQuery(new Term(ArtifactInfo.UINFO, id)), BooleanClause.Occur.MUST)) - .build(); - iterate(repos, (RepositoryInfo repo, IndexingContext context) -> { - IteratorSearchResponse response = repeatedPagedSearch(bq, context, MAX_RESULT_COUNT); - if (response != null) { - try { - for (ArtifactInfo ai : response) { - infos.add(convertToNBVersionInfo(ai)); - } - } finally { - result.addReturnedResultCount(response.getTotalProcessedArtifactInfoCount()); - result.addTotalResultCount(response.getTotalHitsCount()); - response.close(); - } - } - }, skipAction, skipUnIndexed); - doSortIssue226100(infos); - result.setResults(infos); - return result; - } - - @Override - public ResultImplementation getArtifacts(final String groupId, final List repos) { - ResultImpl result = new ResultImpl<>((ResultImpl result1) -> { - getArtifacts(groupId, result1, result1.getSkipped(), false); - }); - return getArtifacts(groupId, result, repos, true); - } - - private ResultImplementation getArtifacts(final String groupId, final ResultImpl result, final List repos, final boolean skipUnIndexed) { - final Set artifacts = new TreeSet<>(result.getResults()); - final SkippedAction skipAction = new SkippedAction(result); - String id = groupId + ArtifactInfo.FS; - BooleanQuery bq = new BooleanQuery.Builder() - .add(new BooleanClause(setBooleanRewrite(new PrefixQuery(new Term(ArtifactInfo.UINFO, id))), BooleanClause.Occur.MUST)) - .build(); - iterate(repos, (RepositoryInfo repo, IndexingContext context) -> { - //mkleint: this is not capped, because only a string is collected (and collapsed), the rest gets CGed fast - IteratorSearchResponse response = repeatedPagedSearch(bq, context, NO_CAP_RESULT_COUNT); - if (response != null) { - try { - for (ArtifactInfo artifactInfo : response.getResults()) { - artifacts.add(artifactInfo.getArtifactId()); - } - } finally { - response.close(); - } - } - }, skipAction, skipUnIndexed); - result.setResults(artifacts); - return result; - } - - @Override - public ResultImplementation getVersions(final String groupId, final String artifactId, List repos) { - ResultImpl result = new ResultImpl<>((ResultImpl result1) -> { - getVersions(groupId, artifactId, result1, result1.getSkipped(), false); - }); - return getVersions(groupId, artifactId, result, repos, true); - } - private ResultImplementation getVersions(final String groupId, final String artifactId, final ResultImpl result, List repos, final boolean skipUnIndexed) { - final List infos = new ArrayList<>(result.getResults()); - final SkippedAction skipAction = new SkippedAction(result); - String id = groupId + ArtifactInfo.FS + artifactId + ArtifactInfo.FS; - BooleanQuery bq = new BooleanQuery.Builder() - .add(new BooleanClause(setBooleanRewrite(new PrefixQuery(new Term(ArtifactInfo.UINFO, id))), BooleanClause.Occur.MUST)) - .build(); - iterate(repos, (RepositoryInfo repo, IndexingContext context) -> { - IteratorSearchResponse response = repeatedPagedSearch(bq, context, MAX_RESULT_COUNT); - if (response != null) { - try { - for (ArtifactInfo ai : response) { - infos.add(convertToNBVersionInfo(ai)); - } - } finally { - result.addReturnedResultCount(response.getTotalProcessedArtifactInfoCount()); - result.addTotalResultCount(response.getTotalHitsCount()); - response.close(); - } - } - }, skipAction, skipUnIndexed); - doSortIssue226100(infos); - result.setResults(infos); - return result; - } - - @Override - public ResultImplementation findVersionsByClass(final String className, List repos) { - - Optional central = repos.stream() - .filter(repo -> repo.getId().equals(getSMO().getRepositoryId())) - .findFirst(); - - // remote index contains no class data -> use web service - if (central.isPresent()) { - List otherRepos = new ArrayList<>(repos); - otherRepos.remove(central.get()); - - SearchRequest request = new SearchRequest(new Paging(128), - FieldQuery.fieldQuery(className.contains(".") ? - org.apache.maven.search.api.MAVEN.FQ_CLASS_NAME - : org.apache.maven.search.api.MAVEN.CLASS_NAME, className)); - - return new CompositeResult<>(findVersionsByClass(className, otherRepos), new SMORequestResult(getSMO(), request)); - } else { - ResultImpl result = new ResultImpl<>((ResultImpl result1) -> { - findVersionsByClass(className, result1, result1.getSkipped(), false); - }); - return findVersionsByClass(className, result, repos, true); - } - } - - private ResultImplementation findVersionsByClass(final String className, final ResultImpl result, List repos, final boolean skipUnIndexed) { - final List infos = new ArrayList<>(result.getResults()); - final SkippedAction skipAction = new SkippedAction(result); - iterate(repos, (RepositoryInfo repo, IndexingContext context) -> { - String clsname = className.replace(".", "/"); - while (!clsname.isEmpty() && (clsname.startsWith("*") || clsname.startsWith("?"))) { - //#238740 - clsname = clsname.substring(1); - } - if (clsname.isEmpty()) { - return; - } - - Query q = setBooleanRewrite(constructQuery(MAVEN.CLASSNAMES, clsname.toLowerCase(Locale.ENGLISH))); - IteratorSearchResponse response = repeatedPagedSearch(q, context, MAX_RESULT_COUNT); - if (response != null) { - try { - infos.addAll(postProcessClasses(response.getResults(), clsname)); - } finally { - //?? really count in this case? - result.addReturnedResultCount(response.getTotalProcessedArtifactInfoCount()); - result.addTotalResultCount(response.getTotalHitsCount()); - response.close(); - } - } - }, skipAction, skipUnIndexed); - doSortIssue226100(infos); - result.setResults(infos); - return result; - } - - private Query constructQuery(Field f, String qs) { - return indexer.constructQuery(f, new StringSearchExpression(qs)); - } - - @Override - public ResultImplementation findClassUsages(final String className, @NullAllowed List repos) { - ResultImpl result = new ResultImpl<>((ResultImpl result1) -> { - findClassUsages(className, result1, result1.getSkipped(), false); - }); - return findClassUsages(className, result, repos, true); - - } - private ResultImplementation findClassUsages(final String className, ResultImpl result, @NullAllowed List repos, final boolean skipUnIndexed) { - List localRepos = new ArrayList<>(); - if (repos == null) { - repos = RepositoryPreferences.getInstance().getRepositoryInfos(); - } - for (RepositoryInfo repo : repos) { - if (repo.isLocal()) { - localRepos.add(repo); - } - } - final List results = new ArrayList<>(result.getResults()); - final SkippedAction skipAction = new SkippedAction(result); - iterate(localRepos, (RepositoryInfo repo, IndexingContext context) -> { - ClassDependencyIndexCreator.search(className, indexer, List.of(context), results); - }, skipAction, skipUnIndexed); - results.sort((RepositoryQueries.ClassUsage r1, RepositoryQueries.ClassUsage r2) -> r1.getArtifact().compareTo(r2.getArtifact())); - result.setResults(results); - return result; - } - - @Override - public ResultImplementation findDependencyUsage(final String groupId, final String artifactId, final String version, @NullAllowed List repos) { - ResultImpl result = new ResultImpl<>((ResultImpl result1) -> { - findDependencyUsage(groupId, artifactId, version, result1, result1.getSkipped(), false); - }); - return findDependencyUsage(groupId, artifactId, version, result, repos, true); - } - private ResultImplementation findDependencyUsage(String groupId, String artifactId, String version, final ResultImpl result, @NullAllowed List repos, final boolean skipUnIndexed) { - final Query q = ArtifactDependencyIndexCreator.query(groupId, artifactId, version); - final List infos = new ArrayList<>(result.getResults()); - final SkippedAction skipAction = new SkippedAction(result); - iterate(repos, (RepositoryInfo repo, IndexingContext context) -> { - IteratorSearchResponse response = repeatedPagedSearch(q, context, MAX_RESULT_COUNT); - if (response != null) { - try { - for (ArtifactInfo ai : response) { - infos.add(convertToNBVersionInfo(ai)); - } - } finally { - result.addReturnedResultCount(response.getTotalProcessedArtifactInfoCount()); - result.addTotalResultCount(response.getTotalHitsCount()); - response.close(); - } - } - }, skipAction, skipUnIndexed); - result.setResults(infos); - return result; - } - - @Override - public ResultImplementation findDependencyUsageGroups(final String groupId, final String artifactId, final String version, List repos) { - ResultImpl result = new ResultImpl<>((ResultImpl result1) -> { - findDependencyUsageGroups(groupId, artifactId, version, result1, result1.getSkipped(), false); - }); - return findDependencyUsageGroups(groupId, artifactId, version, result, repos, true); - } - - private ResultImplementation findDependencyUsageGroups(String groupId, String artifactId, String version, ResultImpl result, List repos, final boolean skipUnIndexed) { - //tempmaps - Map groupMap = new HashMap<>(); - Map artifactMap = new HashMap<>(); - List groupInfos = new ArrayList<>(result.getResults()); - ResultImpl res = new ResultImpl<>((ResultImpl result1) -> { - //noop will not be called - }); - findDependencyUsage(groupId, artifactId, version, res, repos, skipUnIndexed); - convertToNBGroupInfo(res.getResults(), groupMap, artifactMap, groupInfos); - if (res.isPartial()) { - result.addSkipped(res.getSkipped()); - } - result.setResults(groupInfos); - return result; - - } - - private static void convertToNBGroupInfo(Collection artifactInfos, - Map groupMap, - Map artifactMap, - List groupInfos) { - for (NBVersionInfo ai : artifactInfos) { - String groupId = ai.getGroupId(); - String artId = ai.getArtifactId(); - - NBGroupInfo ug = groupMap.get(groupId); - if (ug == null) { - ug = new NBGroupInfo(groupId); - groupInfos.add(ug); - groupMap.put(groupId, ug); - } - NBArtifactInfo ua = artifactMap.get(artId); - if (ua == null) { - ua = new NBArtifactInfo(artId); - ug.addArtifactInfo(ua); - artifactMap.put(artId, ua); - } - ua.addVersionInfo(ai); - } - } - - @Override - public ResultImplementation findBySHA1(final String sha1, List repos) { - - Optional central = repos.stream() - .filter(repo -> repo.getId().equals(getSMO().getRepositoryId())) - .findFirst(); - - // remote index contains no sh1 data -> use web service - if (central.isPresent()) { - List otherRepos = new ArrayList<>(repos); - otherRepos.remove(central.get()); - - SearchRequest request = new SearchRequest(new Paging(8), - FieldQuery.fieldQuery(org.apache.maven.search.api.MAVEN.SHA1, sha1)); - - return new CompositeResult<>(findBySHA1(sha1, otherRepos), new SMORequestResult(getSMO(), request)); - } else { - ResultImpl result = new ResultImpl<>((ResultImpl result1) -> { - findBySHA1(sha1, result1, result1.getSkipped(), false); - }); - return findBySHA1(sha1, result, repos, true); - } - } - - private ResultImplementation findBySHA1(final String sha1, final ResultImpl result, List repos, final boolean skipUnIndexed) { - final List infos = new ArrayList<>(result.getResults()); - final SkippedAction skipAction = new SkippedAction(result); - iterate(repos, (RepositoryInfo repo, IndexingContext context) -> { - BooleanQuery bq = new BooleanQuery.Builder() - .add(new BooleanClause((setBooleanRewrite(constructQuery(MAVEN.SHA1, sha1))), BooleanClause.Occur.SHOULD)) - .build(); - IteratorSearchResponse response = repeatedPagedSearch(bq, context, MAX_RESULT_COUNT); - if (response != null) { - try { - for (ArtifactInfo ai : response) { - infos.add(convertToNBVersionInfo(ai)); - } - } finally { - result.addReturnedResultCount(response.getTotalProcessedArtifactInfoCount()); - result.addTotalResultCount(response.getTotalHitsCount()); - response.close(); - } - } - }, skipAction, skipUnIndexed); - doSortIssue226100(infos); - result.setResults(infos); - return result; - } - - @Override - public ResultImplementation findArchetypes(List repos) { - ResultImpl result = new ResultImpl<>((ResultImpl result1) -> { - findArchetypes(result1, result1.getSkipped(), false); - }); - return findArchetypes( result, repos, true); - } - - private ResultImplementation findArchetypes(final ResultImpl result, List repos, final boolean skipUnIndexed) { - final List infos = new ArrayList<>(result.getResults()); - final SkippedAction skipAction = new SkippedAction(result); - BooleanQuery bq = new BooleanQuery.Builder() - // XXX also consider using NexusArchetypeDataSource - .add(new BooleanClause(new TermQuery(new Term(ArtifactInfo.PACKAGING, "maven-archetype")), BooleanClause.Occur.MUST)) //NOI18N - .build(); - iterate(repos, (RepositoryInfo repo, IndexingContext context) -> { - /* There are >512 archetypes in Central, and we want them all in ChooseArchetypePanel - FlatSearchRequest fsr = new FlatSearchRequest(bq, ArtifactInfo.VERSION_COMPARATOR); - fsr.setCount(MAX_RESULT_COUNT); - */ - IteratorSearchResponse response = repeatedPagedSearch(bq, context, NO_CAP_RESULT_COUNT); - if (response != null) { - try { - for (ArtifactInfo ai : response) { - infos.add(convertToNBVersionInfo(ai)); - } - } finally { - result.addReturnedResultCount(response.getTotalProcessedArtifactInfoCount()); - result.addTotalResultCount(response.getTotalHitsCount()); - response.close(); - } - } - }, skipAction, skipUnIndexed); - doSortIssue226100(infos); - result.setResults(infos); - return result; - } - - @Override - public ResultImplementation filterPluginArtifactIds(final String groupId, final String prefix, List repos) { - ResultImpl result = new ResultImpl<>((ResultImpl result1) -> { - filterPluginArtifactIds(groupId, prefix, result1, result1.getSkipped(), false); - }); - return filterPluginArtifactIds(groupId, prefix, result, repos, true); - } - - private ResultImplementation filterPluginArtifactIds(final String groupId, final String prefix, ResultImpl result, List repos, final boolean skipUnIndexed) { - final Set artifacts = new TreeSet<>(result.getResults()); - final SkippedAction skipAction = new SkippedAction(result); - String id = groupId + ArtifactInfo.FS + prefix; - BooleanQuery bq = new BooleanQuery.Builder() - .add(new BooleanClause(new TermQuery(new Term(ArtifactInfo.PACKAGING, "maven-plugin")), BooleanClause.Occur.MUST)) - .add(new BooleanClause(setBooleanRewrite(new PrefixQuery(new Term(ArtifactInfo.UINFO, id))), BooleanClause.Occur.MUST)) - .build(); - iterate(repos, (RepositoryInfo repo, IndexingContext context) -> { - //mkleint: this is not capped, because only a string is collected (and collapsed), the rest gets CGed fast - IteratorSearchResponse response = repeatedPagedSearch(bq, context, NO_CAP_RESULT_COUNT); - if (response != null) { - try { - for (ArtifactInfo artifactInfo : response.getResults()) { - artifacts.add(artifactInfo.getArtifactId()); - } - } finally { - response.close(); - } - } - }, skipAction, skipUnIndexed); - result.setResults(artifacts); - return result; - } - - @Override - public ResultImplementation filterPluginGroupIds(final String prefix, List repos) { - ResultImpl result = new ResultImpl<>((ResultImpl result1) -> { - filterPluginGroupIds(prefix, result1, result1.getSkipped(), false); - }); - return filterPluginGroupIds( prefix, result, repos, true); - } - - private ResultImplementation filterPluginGroupIds(final String prefix, ResultImpl result, List repos, final boolean skipUnIndexed) { - final Set artifacts = new TreeSet<>(result.getResults()); - final SkippedAction skipAction = new SkippedAction(result); - BooleanQuery.Builder builder = new BooleanQuery.Builder(); - builder.add(new BooleanClause(new TermQuery(new Term(ArtifactInfo.PACKAGING, "maven-plugin")), BooleanClause.Occur.MUST)); - if (!prefix.isEmpty()) { //heap out of memory otherwise - builder.add(new BooleanClause(setBooleanRewrite(new PrefixQuery(new Term(ArtifactInfo.GROUP_ID, prefix))), BooleanClause.Occur.MUST)); - } - BooleanQuery bq = builder.build(); - iterate(repos, (RepositoryInfo repo, IndexingContext context) -> { - //mkleint: this is not capped, because only a string is collected (and collapsed), the rest gets CGed fast - IteratorSearchResponse response = repeatedPagedSearch(bq, context, NO_CAP_RESULT_COUNT); - if (response != null) { - try { - for (ArtifactInfo artifactInfo : response.getResults()) { - artifacts.add(artifactInfo.getGroupId()); - } - } finally { - response.close(); - } - } - }, skipAction, skipUnIndexed); - result.setResults(artifacts); - return result; - } - - @Override - public ResultImplementation find(final List fields, List repos) { - ResultImpl result = new ResultImpl<>((ResultImpl result1) -> { - find(fields, result1, result1.getSkipped(), false); - }); - return find(fields, result, repos, true); - } - private ResultImplementation find(final List fields, final ResultImpl result, List repos, final boolean skipUnIndexed) { - final List infos = new ArrayList<>(result.getResults()); - final SkippedAction skipAction = new SkippedAction(result); - iterate(repos, new RepoAction() { - @Override public void run(RepositoryInfo repo, IndexingContext context) throws IOException { - BooleanQuery.Builder bq = new BooleanQuery.Builder(); - for (QueryField field : fields) { - BooleanClause.Occur occur = field.getOccur() == QueryField.OCCUR_SHOULD ? BooleanClause.Occur.SHOULD : BooleanClause.Occur.MUST; - String fieldName = toNexusField(field.getField()); - String one = field.getValue(); - while (!one.isEmpty() && (one.startsWith("*") || one.startsWith("?"))) { - //#196046 - one = one.substring(1); - } - if (one.isEmpty()) { - continue; - } - - if (fieldName != null) { - Query q; - if (ArtifactInfo.NAMES.equals(fieldName)) { - try { - String clsname = one.replace(".", "/"); //NOI18N - q = constructQuery(MAVEN.CLASSNAMES, clsname.toLowerCase(Locale.ENGLISH)); - } catch (IllegalArgumentException iae) { - //#204651 only escape when problems occur - String clsname = QueryParser.escape(one.replace(".", "/")); //NOI18N - try { - q = constructQuery(MAVEN.CLASSNAMES, clsname.toLowerCase(Locale.ENGLISH)); - } catch (IllegalArgumentException iae2) { - //#224088 - continue; - } - } - } else if (ArtifactInfo.ARTIFACT_ID.equals(fieldName)) { - String aid = one.replace("-", "?").replace(".", "?"); - try { - q = constructQuery(MAVEN.ARTIFACT_ID, aid); - } catch (IllegalArgumentException iae) { - //#204651 only escape when problems occur - try { - q = constructQuery(MAVEN.ARTIFACT_ID, QueryParser.escape(aid)); - } catch (IllegalArgumentException iae2) { - //#224088 - continue; - } - } - } else if (ArtifactInfo.GROUP_ID.equals(fieldName)) { - String gid = one.replace("-", "?").replace(".", "?"); - try { - q = constructQuery(MAVEN.GROUP_ID, gid); - } catch (IllegalArgumentException iae) { - //#204651 only escape when problems occur - try { - q = constructQuery(MAVEN.GROUP_ID, QueryParser.escape(gid)); - } catch (IllegalArgumentException iae2) { - //#224088 - continue; - } - } - } else { - if (field.getMatch() == QueryField.MATCH_EXACT) { - q = new TermQuery(new Term(fieldName, one)); - } else { - q = new PrefixQuery(new Term(fieldName, one)); - } - } - bq.add(new BooleanClause(setBooleanRewrite(q), occur)); - } else { - //TODO when all fields, we need to create separate - //queries for each field. - } - } - IteratorSearchResponse resp = repeatedPagedSearch(bq.build(), context, MAX_RESULT_COUNT); - if (resp != null) { - try { - for (ArtifactInfo ai : resp) { - infos.add(convertToNBVersionInfo(ai)); - } - } finally { - result.addReturnedResultCount(resp.getTotalProcessedArtifactInfoCount()); - result.addTotalResultCount(resp.getTotalHitsCount()); - resp.close(); - } - } - } - }, skipAction, skipUnIndexed); - doSortIssue226100(infos); - result.setResults(infos); - return result; - } - - private void doSortIssue226100(List infos) { - try { - Collections.sort(infos); - } catch (IllegalStateException | IllegalArgumentException ex) { -// doLogError226100(infos, ex); - } - } - - private void doLogError226100(List infos, Exception ex) throws RuntimeException { - //#226100 - StringBuilder versions = new StringBuilder(); - for (NBVersionInfo info : infos) { - versions.append(info.getVersion()).append(","); - } - String message = "Issue #226100: Versions compared are:" + versions.toString(); - LOGGER.log(Level.WARNING, message); - boolean rethrow = false; - assert rethrow = true == false; - if (rethrow) { - throw new RuntimeException( message, ex); - } - } - - @Override - public List getLoaded(final List repos) { - final List toRet = new ArrayList<>(repos.size()); - for (final RepositoryInfo repo : repos) { - Path loc = getIndexDirectory(repo); // index folder - if (indexExists(loc)) { - toRet.add(repo); - } - } - return toRet; - } - - private static boolean indexExists(Path path) { + static boolean indexExists(Path path) { try { return Files.exists(path.resolve("timestamp")) && DirectoryReader.indexExists(new MMapDirectory(path)); } catch (IOException ex) { @@ -1666,106 +982,7 @@ private static boolean indexExists(Path path) { return false; } } - - private String toNexusField(String field) { - if (field != null) switch (field) { - case QueryField.FIELD_ARTIFACTID: return ArtifactInfo.ARTIFACT_ID; - case QueryField.FIELD_GROUPID: return ArtifactInfo.GROUP_ID; - case QueryField.FIELD_VERSION: return ArtifactInfo.VERSION; - case QueryField.FIELD_CLASSES: return ArtifactInfo.NAMES; - case QueryField.FIELD_NAME: return ArtifactInfo.NAME; - case QueryField.FIELD_DESCRIPTION: return ArtifactInfo.DESCRIPTION; - case QueryField.FIELD_PACKAGING: return ArtifactInfo.PACKAGING; - } - return field; - } - - private Collection postProcessClasses(IteratorResultSet artifactInfos, String classname) { - List toRet = new ArrayList<>(); - int patter = Pattern.DOTALL + Pattern.MULTILINE; - boolean isPath = classname.contains("/"); - if (isPath) { - for (ArtifactInfo i : artifactInfos) { - toRet.add(convertToNBVersionInfo(i)); - } - return toRet; - } - //if I got it right, we need an exact match of class name, which the query doesn't provide? why? - String pattStr = ".*/" + classname + "$.*"; - Pattern patt = Pattern.compile(pattStr, patter); - //#217932 for some reason IteratorResultSet implementation decided - //not to implemenent Iterator.remove(). - //we need to copy to our own list instead of removing from original. - ArrayList altArtifactInfos = new ArrayList<>(); - for (ArtifactInfo ai : artifactInfos) { - Matcher m = patt.matcher(ai.getClassNames()); - if (m.matches()) { - altArtifactInfos.add(ai); - } - } - for (ArtifactInfo i : altArtifactInfos) { - toRet.add(convertToNBVersionInfo(i)); - } - return toRet; - } - - static List convertToNBVersionInfo(Collection artifactInfos) { - List bVersionInfos = new ArrayList<>(); - for (ArtifactInfo ai : artifactInfos) { - NBVersionInfo nbvi = convertToNBVersionInfo(ai); - if (nbvi != null) { - bVersionInfos.add(nbvi); - } - } - return bVersionInfos; - } - static NBVersionInfo convertToNBVersionInfo(ArtifactInfo ai) { - if ("javadoc".equals(ai.getClassifier()) || "sources".equals(ai.getClassifier())) { //NOI18N - // we don't want javadoc and sources shown anywhere, we use the getJavadocExists(), getSourceExists() methods. - return null; - } - // fextension != packaging - e.g a pom could be packaging "bundle" but from type/extension "jar" - NBVersionInfo nbvi = new NBVersionInfo(ai.getRepository(), ai.getGroupId(), ai.getArtifactId(), - ai.getVersion(), ai.getFileExtension(), ai.getPackaging(), ai.getName(), ai.getDescription(), ai.getClassifier()); - /*Javadoc & Sources*/ - nbvi.setJavadocExists(ai.getJavadocExists() == ArtifactAvailability.PRESENT); - nbvi.setSourcesExists(ai.getSourcesExists() == ArtifactAvailability.PRESENT); - nbvi.setSignatureExists(ai.getSignatureExists() == ArtifactAvailability.PRESENT); -// nbvi.setSha(ai.sha1); - nbvi.setLastModified(ai.getLastModified()); - nbvi.setSize(ai.getSize()); - nbvi.setLuceneScore(ai.getLuceneScore()); - return nbvi; - } - - private static Query setBooleanRewrite (final Query q) { - if (q instanceof MultiTermQuery multiTermQuery) { - multiTermQuery.setRewriteMethod(MultiTermQuery.CONSTANT_SCORE_REWRITE); - } else if (q instanceof BooleanQuery booleanQuery) { - for (BooleanClause c : booleanQuery.clauses()) { - setBooleanRewrite(c.getQuery()); - } - } - return q; - } - - private record SkippedAction(ResultImpl result) implements RepoAction { - @Override - public void run(RepositoryInfo repo, IndexingContext context) throws IOException { - //indexing context is always null here.. - result.addSkipped(repo); - } - } - - private static class NoJavadocSourceFilter implements ArtifactInfoFilter { - - @Override - public boolean accepts(IndexingContext ctx, ArtifactInfo ai) { - return !("javadoc".equals(ai.getClassifier()) || "sources".equals(ai.getClassifier())); - } - - } - + private ResourceFetcher createFetcher(final Wagon wagon, TransferListener listener, AuthenticationInfo authenticationInfo, ProxyInfo proxyInfo) { return new WagonFetcher(wagon, listener, authenticationInfo, proxyInfo); } @@ -1774,10 +991,14 @@ private static Path getIndexDirectory() { return Places.getCacheSubdirectory("mavenindex").toPath(); } - private static Path getIndexDirectory(final RepositoryInfo info) { + static Path getIndexDirectory(final RepositoryInfo info) { return getIndexDirectory().resolve(info.getId()); } + private static Path getIndexDirectory(Path cache, RepositoryInfo info) { + return cache.resolve("mavenindex").resolve(info.getId()); + } + private static Path getAllGroupCacheFile(final RepositoryInfo info) { return getIndexDirectory(info).resolve(GROUP_CACHE_ALL_PREFIX + "." + GROUP_CACHE_ALL_SUFFIX); } @@ -1829,16 +1050,12 @@ private static void rebuildGroupCache(RepositoryInfo repo, IndexingContext conte }); } - private synchronized SmoSearchBackend getSMO() { - if (smo == null) { - smo = SmoSearchBackendFactory.create( - SmoSearchBackendFactory.DEFAULT_BACKEND_ID, - SmoSearchBackendFactory.DEFAULT_REPOSITORY_ID, - SmoSearchBackendFactory.DEFAULT_SMO_URI, - new Java11HttpClientTransport(SMORequestResult.REQUEST_TIMEOUT) - ); - } - return smo; + Indexer getIndexer() { + return indexer; + } + + SearchEngine getSearcher() { + return searcher; } // somewhat based on maven-indexer impl (in WagonHelper) prior to removal in maven-indexer 7.0.0 diff --git a/java/maven.indexer/src/org/netbeans/modules/maven/indexer/NexusRepositoryQueries.java b/java/maven.indexer/src/org/netbeans/modules/maven/indexer/NexusRepositoryQueries.java new file mode 100644 index 000000000000..2a82c659c19e --- /dev/null +++ b/java/maven.indexer/src/org/netbeans/modules/maven/indexer/NexusRepositoryQueries.java @@ -0,0 +1,918 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.netbeans.modules.maven.indexer; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.TreeSet; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.apache.lucene.index.Term; +import org.apache.lucene.queryparser.classic.QueryParser; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MultiTermQuery; +import org.apache.lucene.search.PrefixQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; +import org.apache.maven.index.ArtifactAvailability; +import org.apache.maven.index.ArtifactInfo; +import org.apache.maven.index.ArtifactInfoFilter; +import org.apache.maven.index.Field; +import org.apache.maven.index.IteratorResultSet; +import org.apache.maven.index.IteratorSearchRequest; +import org.apache.maven.index.IteratorSearchResponse; +import org.apache.maven.index.MAVEN; +import org.apache.maven.index.context.IndexingContext; +import org.apache.maven.index.expr.StringSearchExpression; +import org.apache.maven.search.api.SearchRequest; +import org.apache.maven.search.api.request.FieldQuery; +import org.apache.maven.search.api.request.Paging; +import org.apache.maven.search.api.transport.Java11HttpClientTransport; +import org.apache.maven.search.backend.smo.SmoSearchBackend; +import org.apache.maven.search.backend.smo.SmoSearchBackendFactory; +import org.netbeans.api.annotations.common.CheckForNull; +import org.netbeans.api.annotations.common.NullAllowed; +import org.netbeans.modules.maven.indexer.api.NBArtifactInfo; +import org.netbeans.modules.maven.indexer.api.NBGroupInfo; +import org.netbeans.modules.maven.indexer.api.NBVersionInfo; +import org.netbeans.modules.maven.indexer.api.QueryField; +import org.netbeans.modules.maven.indexer.api.RepositoryInfo; +import org.netbeans.modules.maven.indexer.api.RepositoryPreferences; +import org.netbeans.modules.maven.indexer.api.RepositoryQueries; +import org.netbeans.modules.maven.indexer.spi.ArchetypeQueries; +import org.netbeans.modules.maven.indexer.spi.BaseQueries; +import org.netbeans.modules.maven.indexer.spi.ChecksumQueries; +import org.netbeans.modules.maven.indexer.spi.ClassUsageQuery; +import org.netbeans.modules.maven.indexer.spi.ClassesQuery; +import org.netbeans.modules.maven.indexer.spi.ContextLoadedQuery; +import org.netbeans.modules.maven.indexer.spi.DependencyInfoQueries; +import org.netbeans.modules.maven.indexer.spi.GenericFindQuery; +import org.netbeans.modules.maven.indexer.spi.ResultImplementation; +import org.openide.util.Mutex; + +/** + * Implements all maven queries. + */ +final class NexusRepositoryQueries implements + BaseQueries, ChecksumQueries, ArchetypeQueries, DependencyInfoQueries, + ClassesQuery, ClassUsageQuery, GenericFindQuery, ContextLoadedQuery { + + private static final Logger LOGGER = Logger.getLogger(NexusRepositoryQueries.class.getName()); + + private SmoSearchBackend smo; + private final NexusRepositoryIndexManager indexer; + + NexusRepositoryQueries(NexusRepositoryIndexManager indexer) { + this.indexer = indexer; + } + + @Override + public ResultImplementation getGroups(List repos) { + return filterGroupIds("", repos); + } + + private ResultImplementation filterGroupIds(String prefix, List repos) { + ResultImpl result = new ResultImpl<>( + (ResultImpl res) -> filterGroupIds(prefix, res, res.getSkipped(), false) + ); + return filterGroupIds(prefix, result, repos, true); + } + + private ResultImplementation filterGroupIds(String prefix, ResultImpl result, List repos, boolean skipUnIndexed) { + final Set groups = new TreeSet<>(result.getResults()); + final SkippedAction skipAction = new SkippedAction(result); + + iterate(repos, (RepositoryInfo repo, IndexingContext context) -> { + Set all= context.getAllGroups(); + if (!all.isEmpty()) { + if (prefix.isEmpty()) { + groups.addAll(all); + } else { + for (String gr : all) { + if (gr.startsWith(prefix)) { + groups.add(gr); + } + } + } + } + }, skipAction, skipUnIndexed); + + result.setResults(groups); + return result; + } + + @Override + public ResultImplementation getGAVsForPackaging(String packaging, List repos) { + ResultImpl result = new ResultImpl<>((ResultImpl result1) -> { + getGAVsForPackaging(packaging, result1, result1.getSkipped(), false); + }); + return getGAVsForPackaging(packaging,result, repos, true); + } + + private ResultImplementation getGAVsForPackaging(String packaging, ResultImpl result, List repos, boolean skipUnIndexed) { + final List infos = new ArrayList<>(result.getResults()); + final SkippedAction skipAction = new SkippedAction(result); + BooleanQuery bq = new BooleanQuery.Builder() + .add(new BooleanClause(new TermQuery(new Term(ArtifactInfo.PACKAGING, packaging)), BooleanClause.Occur.MUST)) + .build(); + iterate(repos, (RepositoryInfo repo, IndexingContext context) -> { + IteratorSearchResponse response = repeatedPagedSearch(bq, context, NexusRepositoryIndexManager.NO_CAP_RESULT_COUNT); + if (response != null) { + try { + for (ArtifactInfo ai : response) { + String gav = ai.getGroupId() + ":" + ai.getArtifactId() + ":" + ai.getVersion(); + if (!infos.contains(gav)) { + infos.add(gav); + } + } + } finally { + result.addReturnedResultCount(response.getTotalProcessedArtifactInfoCount()); + result.addTotalResultCount(response.getTotalHitsCount()); + response.close(); + } + } + }, skipAction, skipUnIndexed); + result.setResults(infos); + return result; + } + + @Override + public ResultImplementation getRecords(String groupId, String artifactId, String version, List repos) { + ResultImpl result = new ResultImpl<>((ResultImpl result1) -> { + getRecords(groupId, artifactId, version, result1, result1.getSkipped(), false); + }); + return getRecords(groupId, artifactId, version, result, repos, true); + } + + private ResultImplementation getRecords(String groupId, String artifactId, String version, ResultImpl result, + List repos, final boolean skipUnIndexed) { + final List infos = new ArrayList<>(result.getResults()); + final SkippedAction skipAction = new SkippedAction(result); + String id = groupId + ArtifactInfo.FS + artifactId + ArtifactInfo.FS + version + ArtifactInfo.FS; + BooleanQuery bq = new BooleanQuery.Builder() + .add(new BooleanClause(new PrefixQuery(new Term(ArtifactInfo.UINFO, id)), BooleanClause.Occur.MUST)) + .build(); + iterate(repos, (RepositoryInfo repo, IndexingContext context) -> { + IteratorSearchResponse response = repeatedPagedSearch(bq, context, NexusRepositoryIndexManager.MAX_RESULT_COUNT); + if (response != null) { + try { + for (ArtifactInfo ai : response) { + infos.add(convertToNBVersionInfo(ai)); + } + } finally { + result.addReturnedResultCount(response.getTotalProcessedArtifactInfoCount()); + result.addTotalResultCount(response.getTotalHitsCount()); + response.close(); + } + } + }, skipAction, skipUnIndexed); + doSortIssue226100(infos); + result.setResults(infos); + return result; + } + + @Override + public ResultImplementation getArtifacts(String groupId, List repos) { + ResultImpl result = new ResultImpl<>((ResultImpl result1) -> { + getArtifacts(groupId, result1, result1.getSkipped(), false); + }); + return getArtifacts(groupId, result, repos, true); + } + + private ResultImplementation getArtifacts(String groupId, ResultImpl result, List repos, boolean skipUnIndexed) { + final Set artifacts = new TreeSet<>(result.getResults()); + final SkippedAction skipAction = new SkippedAction(result); + String id = groupId + ArtifactInfo.FS; + BooleanQuery bq = new BooleanQuery.Builder() + .add(new BooleanClause(setBooleanRewrite(new PrefixQuery(new Term(ArtifactInfo.UINFO, id))), BooleanClause.Occur.MUST)) + .build(); + iterate(repos, (RepositoryInfo repo, IndexingContext context) -> { + //mkleint: this is not capped, because only a string is collected (and collapsed), the rest gets CGed fast + IteratorSearchResponse response = repeatedPagedSearch(bq, context, NexusRepositoryIndexManager.NO_CAP_RESULT_COUNT); + if (response != null) { + try { + for (ArtifactInfo artifactInfo : response.getResults()) { + artifacts.add(artifactInfo.getArtifactId()); + } + } finally { + response.close(); + } + } + }, skipAction, skipUnIndexed); + result.setResults(artifacts); + return result; + } + + @Override + public ResultImplementation getVersions(String groupId, String artifactId, List repos) { + ResultImpl result = new ResultImpl<>((ResultImpl result1) -> { + getVersions(groupId, artifactId, result1, result1.getSkipped(), false); + }); + return getVersions(groupId, artifactId, result, repos, true); + } + + private ResultImplementation getVersions(String groupId, String artifactId, ResultImpl result, List repos, boolean skipUnIndexed) { + final List infos = new ArrayList<>(result.getResults()); + final SkippedAction skipAction = new SkippedAction(result); + String id = groupId + ArtifactInfo.FS + artifactId + ArtifactInfo.FS; + BooleanQuery bq = new BooleanQuery.Builder() + .add(new BooleanClause(setBooleanRewrite(new PrefixQuery(new Term(ArtifactInfo.UINFO, id))), BooleanClause.Occur.MUST)) + .build(); + iterate(repos, (RepositoryInfo repo, IndexingContext context) -> { + IteratorSearchResponse response = repeatedPagedSearch(bq, context, NexusRepositoryIndexManager.MAX_RESULT_COUNT); + if (response != null) { + try { + for (ArtifactInfo ai : response) { + infos.add(convertToNBVersionInfo(ai)); + } + } finally { + result.addReturnedResultCount(response.getTotalProcessedArtifactInfoCount()); + result.addTotalResultCount(response.getTotalHitsCount()); + response.close(); + } + } + }, skipAction, skipUnIndexed); + doSortIssue226100(infos); + result.setResults(infos); + return result; + } + + @Override + public ResultImplementation findVersionsByClass(String className, List repos) { + + Optional central = repos.stream() + .filter(repo -> repo.getId().equals(getSMO().getRepositoryId())) + .findFirst(); + + // remote index contains no class data -> use web service + if (central.isPresent()) { + List otherRepos = new ArrayList<>(repos); + otherRepos.remove(central.get()); + + SearchRequest request = new SearchRequest(new Paging(128), + FieldQuery.fieldQuery(className.contains(".") ? + org.apache.maven.search.api.MAVEN.FQ_CLASS_NAME + : org.apache.maven.search.api.MAVEN.CLASS_NAME, className)); + + return new CompositeResult<>(findVersionsByClass(className, otherRepos), new SMORequestResult(getSMO(), request)); + } else { + ResultImpl result = new ResultImpl<>((ResultImpl result1) -> { + findVersionsByClass(className, result1, result1.getSkipped(), false); + }); + return findVersionsByClass(className, result, repos, true); + } + } + + private ResultImplementation findVersionsByClass(String className, ResultImpl result, List repos, boolean skipUnIndexed) { + final List infos = new ArrayList<>(result.getResults()); + final SkippedAction skipAction = new SkippedAction(result); + iterate(repos, (RepositoryInfo repo, IndexingContext context) -> { + String clsname = className.replace(".", "/"); + while (!clsname.isEmpty() && (clsname.startsWith("*") || clsname.startsWith("?"))) { + //#238740 + clsname = clsname.substring(1); + } + if (clsname.isEmpty()) { + return; + } + + Query q = setBooleanRewrite(constructQuery(MAVEN.CLASSNAMES, clsname.toLowerCase(Locale.ENGLISH))); + IteratorSearchResponse response = repeatedPagedSearch(q, context, NexusRepositoryIndexManager.MAX_RESULT_COUNT); + if (response != null) { + try { + infos.addAll(postProcessClasses(response.getResults(), clsname)); + } finally { + //?? really count in this case? + result.addReturnedResultCount(response.getTotalProcessedArtifactInfoCount()); + result.addTotalResultCount(response.getTotalHitsCount()); + response.close(); + } + } + }, skipAction, skipUnIndexed); + doSortIssue226100(infos); + result.setResults(infos); + return result; + } + + private Collection postProcessClasses(IteratorResultSet artifactInfos, String classname) { + List toRet = new ArrayList<>(); + int patter = Pattern.DOTALL + Pattern.MULTILINE; + boolean isPath = classname.contains("/"); + if (isPath) { + for (ArtifactInfo i : artifactInfos) { + toRet.add(convertToNBVersionInfo(i)); + } + return toRet; + } + //if I got it right, we need an exact match of class name, which the query doesn't provide? why? + String pattStr = ".*/" + classname + "$.*"; + Pattern patt = Pattern.compile(pattStr, patter); + //#217932 for some reason IteratorResultSet implementation decided + //not to implemenent Iterator.remove(). + //we need to copy to our own list instead of removing from original. + ArrayList altArtifactInfos = new ArrayList<>(); + for (ArtifactInfo ai : artifactInfos) { + Matcher m = patt.matcher(ai.getClassNames()); + if (m.matches()) { + altArtifactInfos.add(ai); + } + } + for (ArtifactInfo i : altArtifactInfos) { + toRet.add(convertToNBVersionInfo(i)); + } + return toRet; + } + + @Override + public ResultImplementation findClassUsages(String className, @NullAllowed List repos) { + ResultImpl result = new ResultImpl<>((ResultImpl result1) -> { + findClassUsages(className, result1, result1.getSkipped(), false); + }); + return findClassUsages(className, result, repos, true); + } + + @SuppressWarnings("AssignmentToMethodParameter") + private ResultImplementation findClassUsages(String className, ResultImpl result, @NullAllowed List repos, boolean skipUnIndexed) { + List localRepos = new ArrayList<>(); + if (repos == null) { + repos = RepositoryPreferences.getInstance().getRepositoryInfos(); + } + for (RepositoryInfo repo : repos) { + if (repo.isLocal()) { + localRepos.add(repo); + } + } + final List results = new ArrayList<>(result.getResults()); + final SkippedAction skipAction = new SkippedAction(result); + iterate(localRepos, (RepositoryInfo repo, IndexingContext context) -> { + ClassDependencyIndexCreator.search(className, indexer.getIndexer(), List.of(context), results); + }, skipAction, skipUnIndexed); + results.sort((RepositoryQueries.ClassUsage r1, RepositoryQueries.ClassUsage r2) -> r1.getArtifact().compareTo(r2.getArtifact())); + result.setResults(results); + return result; + } + + @Override + public ResultImplementation findDependencyUsage(String groupId, String artifactId, String version, @NullAllowed List repos) { + ResultImpl result = new ResultImpl<>((ResultImpl result1) -> { + findDependencyUsage(groupId, artifactId, version, result1, result1.getSkipped(), false); + }); + return findDependencyUsage(groupId, artifactId, version, result, repos, true); + } + + private ResultImplementation findDependencyUsage(String groupId, String artifactId, String version, ResultImpl result, @NullAllowed List repos, boolean skipUnIndexed) { + final Query q = ArtifactDependencyIndexCreator.query(groupId, artifactId, version); + final List infos = new ArrayList<>(result.getResults()); + final SkippedAction skipAction = new SkippedAction(result); + iterate(repos, (RepositoryInfo repo, IndexingContext context) -> { + IteratorSearchResponse response = repeatedPagedSearch(q, context, NexusRepositoryIndexManager.MAX_RESULT_COUNT); + if (response != null) { + try { + for (ArtifactInfo ai : response) { + infos.add(convertToNBVersionInfo(ai)); + } + } finally { + result.addReturnedResultCount(response.getTotalProcessedArtifactInfoCount()); + result.addTotalResultCount(response.getTotalHitsCount()); + response.close(); + } + } + }, skipAction, skipUnIndexed); + result.setResults(infos); + return result; + } + + @Override + public ResultImplementation findDependencyUsageGroups(String groupId, String artifactId, String version, List repos) { + ResultImpl result = new ResultImpl<>((ResultImpl result1) -> { + findDependencyUsageGroups(groupId, artifactId, version, result1, result1.getSkipped(), false); + }); + return findDependencyUsageGroups(groupId, artifactId, version, result, repos, true); + } + + private ResultImplementation findDependencyUsageGroups(String groupId, String artifactId, String version, ResultImpl result, List repos, final boolean skipUnIndexed) { + //tempmaps + Map groupMap = new HashMap<>(); + Map artifactMap = new HashMap<>(); + List groupInfos = new ArrayList<>(result.getResults()); + ResultImpl res = new ResultImpl<>((ResultImpl result1) -> { + //noop will not be called + }); + findDependencyUsage(groupId, artifactId, version, res, repos, skipUnIndexed); + convertToNBGroupInfo(res.getResults(), groupMap, artifactMap, groupInfos); + if (res.isPartial()) { + result.addSkipped(res.getSkipped()); + } + result.setResults(groupInfos); + return result; + } + + private static void convertToNBGroupInfo(Collection artifactInfos, + Map groupMap, + Map artifactMap, + List groupInfos) { + for (NBVersionInfo ai : artifactInfos) { + String groupId = ai.getGroupId(); + String artId = ai.getArtifactId(); + + NBGroupInfo ug = groupMap.get(groupId); + if (ug == null) { + ug = new NBGroupInfo(groupId); + groupInfos.add(ug); + groupMap.put(groupId, ug); + } + NBArtifactInfo ua = artifactMap.get(artId); + if (ua == null) { + ua = new NBArtifactInfo(artId); + ug.addArtifactInfo(ua); + artifactMap.put(artId, ua); + } + ua.addVersionInfo(ai); + } + } + + @Override + public ResultImplementation findBySHA1(String sha1, List repos) { + + Optional central = repos.stream() + .filter(repo -> repo.getId().equals(getSMO().getRepositoryId())) + .findFirst(); + + // remote index contains no sh1 data -> use web service + if (central.isPresent()) { + List otherRepos = new ArrayList<>(repos); + otherRepos.remove(central.get()); + + SearchRequest request = new SearchRequest(new Paging(8), + FieldQuery.fieldQuery(org.apache.maven.search.api.MAVEN.SHA1, sha1)); + + return new CompositeResult<>(findBySHA1(sha1, otherRepos), new SMORequestResult(getSMO(), request)); + } else { + ResultImpl result = new ResultImpl<>((ResultImpl result1) -> { + findBySHA1(sha1, result1, result1.getSkipped(), false); + }); + return findBySHA1(sha1, result, repos, true); + } + } + + private ResultImplementation findBySHA1(String sha1, ResultImpl result, List repos, boolean skipUnIndexed) { + final List infos = new ArrayList<>(result.getResults()); + final SkippedAction skipAction = new SkippedAction(result); + iterate(repos, (RepositoryInfo repo, IndexingContext context) -> { + BooleanQuery bq = new BooleanQuery.Builder() + .add(new BooleanClause((setBooleanRewrite(constructQuery(MAVEN.SHA1, sha1))), BooleanClause.Occur.SHOULD)) + .build(); + IteratorSearchResponse response = repeatedPagedSearch(bq, context, NexusRepositoryIndexManager.MAX_RESULT_COUNT); + if (response != null) { + try { + for (ArtifactInfo ai : response) { + infos.add(convertToNBVersionInfo(ai)); + } + } finally { + result.addReturnedResultCount(response.getTotalProcessedArtifactInfoCount()); + result.addTotalResultCount(response.getTotalHitsCount()); + response.close(); + } + } + }, skipAction, skipUnIndexed); + doSortIssue226100(infos); + result.setResults(infos); + return result; + } + + @Override + public ResultImplementation findArchetypes(List repos) { + ResultImpl result = new ResultImpl<>((ResultImpl result1) -> { + findArchetypes(result1, result1.getSkipped(), false); + }); + return findArchetypes( result, repos, true); + } + + private ResultImplementation findArchetypes(ResultImpl result, List repos, boolean skipUnIndexed) { + final List infos = new ArrayList<>(result.getResults()); + final SkippedAction skipAction = new SkippedAction(result); + BooleanQuery bq = new BooleanQuery.Builder() + // XXX also consider using NexusArchetypeDataSource + .add(new BooleanClause(new TermQuery(new Term(ArtifactInfo.PACKAGING, "maven-archetype")), BooleanClause.Occur.MUST)) //NOI18N + .build(); + iterate(repos, (RepositoryInfo repo, IndexingContext context) -> { + /* There are >512 archetypes in Central, and we want them all in ChooseArchetypePanel + FlatSearchRequest fsr = new FlatSearchRequest(bq, ArtifactInfo.VERSION_COMPARATOR); + fsr.setCount(NexusRepositoryIndexerImpl.MAX_RESULT_COUNT); + */ + IteratorSearchResponse response = repeatedPagedSearch(bq, context, NexusRepositoryIndexManager.NO_CAP_RESULT_COUNT); + if (response != null) { + try { + for (ArtifactInfo ai : response) { + infos.add(convertToNBVersionInfo(ai)); + } + } finally { + result.addReturnedResultCount(response.getTotalProcessedArtifactInfoCount()); + result.addTotalResultCount(response.getTotalHitsCount()); + response.close(); + } + } + }, skipAction, skipUnIndexed); + doSortIssue226100(infos); + result.setResults(infos); + return result; + } + + @Override + public ResultImplementation filterPluginArtifactIds(String groupId, String prefix, List repos) { + ResultImpl result = new ResultImpl<>((ResultImpl result1) -> { + filterPluginArtifactIds(groupId, prefix, result1, result1.getSkipped(), false); + }); + return filterPluginArtifactIds(groupId, prefix, result, repos, true); + } + + private ResultImplementation filterPluginArtifactIds(String groupId, String prefix, ResultImpl result, List repos, boolean skipUnIndexed) { + final Set artifacts = new TreeSet<>(result.getResults()); + final SkippedAction skipAction = new SkippedAction(result); + String id = groupId + ArtifactInfo.FS + prefix; + BooleanQuery bq = new BooleanQuery.Builder() + .add(new BooleanClause(new TermQuery(new Term(ArtifactInfo.PACKAGING, "maven-plugin")), BooleanClause.Occur.MUST)) + .add(new BooleanClause(setBooleanRewrite(new PrefixQuery(new Term(ArtifactInfo.UINFO, id))), BooleanClause.Occur.MUST)) + .build(); + iterate(repos, (RepositoryInfo repo, IndexingContext context) -> { + //mkleint: this is not capped, because only a string is collected (and collapsed), the rest gets CGed fast + IteratorSearchResponse response = repeatedPagedSearch(bq, context, NexusRepositoryIndexManager.NO_CAP_RESULT_COUNT); + if (response != null) { + try { + for (ArtifactInfo artifactInfo : response.getResults()) { + artifacts.add(artifactInfo.getArtifactId()); + } + } finally { + response.close(); + } + } + }, skipAction, skipUnIndexed); + result.setResults(artifacts); + return result; + } + + @Override + public ResultImplementation filterPluginGroupIds(String prefix, List repos) { + ResultImpl result = new ResultImpl<>((ResultImpl result1) -> { + filterPluginGroupIds(prefix, result1, result1.getSkipped(), false); + }); + return filterPluginGroupIds( prefix, result, repos, true); + } + + private ResultImplementation filterPluginGroupIds(String prefix, ResultImpl result, List repos, boolean skipUnIndexed) { + final Set artifacts = new TreeSet<>(result.getResults()); + final SkippedAction skipAction = new SkippedAction(result); + BooleanQuery.Builder builder = new BooleanQuery.Builder(); + builder.add(new BooleanClause(new TermQuery(new Term(ArtifactInfo.PACKAGING, "maven-plugin")), BooleanClause.Occur.MUST)); + if (!prefix.isEmpty()) { //heap out of memory otherwise + builder.add(new BooleanClause(setBooleanRewrite(new PrefixQuery(new Term(ArtifactInfo.GROUP_ID, prefix))), BooleanClause.Occur.MUST)); + } + BooleanQuery bq = builder.build(); + iterate(repos, (RepositoryInfo repo, IndexingContext context) -> { + //mkleint: this is not capped, because only a string is collected (and collapsed), the rest gets CGed fast + IteratorSearchResponse response = repeatedPagedSearch(bq, context, NexusRepositoryIndexManager.NO_CAP_RESULT_COUNT); + if (response != null) { + try { + for (ArtifactInfo artifactInfo : response.getResults()) { + artifacts.add(artifactInfo.getGroupId()); + } + } finally { + response.close(); + } + } + }, skipAction, skipUnIndexed); + result.setResults(artifacts); + return result; + } + + @Override + public ResultImplementation find(final List fields, List repos) { + ResultImpl result = new ResultImpl<>((ResultImpl result1) -> { + find(fields, result1, result1.getSkipped(), false); + }); + return find(fields, result, repos, true); + } + + private ResultImplementation find(List fields, ResultImpl result, List repos, boolean skipUnIndexed) { + final List infos = new ArrayList<>(result.getResults()); + final SkippedAction skipAction = new SkippedAction(result); + iterate(repos, (RepositoryInfo repo, IndexingContext context) -> { + BooleanQuery.Builder bq = new BooleanQuery.Builder(); + for (QueryField field : fields) { + BooleanClause.Occur occur = field.getOccur() == QueryField.OCCUR_SHOULD ? BooleanClause.Occur.SHOULD : BooleanClause.Occur.MUST; + String fieldName = toNexusField(field.getField()); + String one = field.getValue(); + while (!one.isEmpty() && (one.startsWith("*") || one.startsWith("?"))) { + //#196046 + one = one.substring(1); + } + if (one.isEmpty()) { + continue; + } + + if (fieldName != null) { + Query q; + if (ArtifactInfo.NAMES.equals(fieldName)) { + try { + String clsname = one.replace(".", "/"); //NOI18N + q = constructQuery(MAVEN.CLASSNAMES, clsname.toLowerCase(Locale.ENGLISH)); + } catch (IllegalArgumentException iae) { + //#204651 only escape when problems occur + String clsname = QueryParser.escape(one.replace(".", "/")); //NOI18N + try { + q = constructQuery(MAVEN.CLASSNAMES, clsname.toLowerCase(Locale.ENGLISH)); + } catch (IllegalArgumentException iae2) { + //#224088 + continue; + } + } + } else if (ArtifactInfo.ARTIFACT_ID.equals(fieldName)) { + String aid = one.replace("-", "?").replace(".", "?"); + try { + q = constructQuery(MAVEN.ARTIFACT_ID, aid); + } catch (IllegalArgumentException iae) { + //#204651 only escape when problems occur + try { + q = constructQuery(MAVEN.ARTIFACT_ID, QueryParser.escape(aid)); + } catch (IllegalArgumentException iae2) { + //#224088 + continue; + } + } + } else if (ArtifactInfo.GROUP_ID.equals(fieldName)) { + String gid = one.replace("-", "?").replace(".", "?"); + try { + q = constructQuery(MAVEN.GROUP_ID, gid); + } catch (IllegalArgumentException iae) { + //#204651 only escape when problems occur + try { + q = constructQuery(MAVEN.GROUP_ID, QueryParser.escape(gid)); + } catch (IllegalArgumentException iae2) { + //#224088 + continue; + } + } + } else { + if (field.getMatch() == QueryField.MATCH_EXACT) { + q = new TermQuery(new Term(fieldName, one)); + } else { + q = new PrefixQuery(new Term(fieldName, one)); + } + } + bq.add(new BooleanClause(setBooleanRewrite(q), occur)); + } else { + //TODO when all fields, we need to create separate + //queries for each field. + } + } + IteratorSearchResponse resp = repeatedPagedSearch(bq.build(), context, NexusRepositoryIndexManager.MAX_RESULT_COUNT); + if (resp != null) { + try { + for (ArtifactInfo ai : resp) { + infos.add(convertToNBVersionInfo(ai)); + } + } finally { + result.addReturnedResultCount(resp.getTotalProcessedArtifactInfoCount()); + result.addTotalResultCount(resp.getTotalHitsCount()); + resp.close(); + } + } + }, skipAction, skipUnIndexed); + doSortIssue226100(infos); + result.setResults(infos); + return result; + } + + private void doSortIssue226100(List infos) { + try { + Collections.sort(infos); + } catch (IllegalStateException | IllegalArgumentException ex) { +// doLogError226100(infos, ex); + } + } + + @SuppressWarnings("AssignmentToMethodParameter") + private void iterate(List repos, RepoAction action, RepoAction actionSkip, boolean skipUnIndexed) { + if (repos == null) { + repos = RepositoryPreferences.getInstance().getRepositoryInfos(); + } + for (final RepositoryInfo repo : repos) { + Mutex mutex = NexusRepositoryIndexManager.getRepoMutex(repo); + if (skipUnIndexed && NexusRepositoryIndexManager.isIndexing(mutex)) { + try { + actionSkip.run(repo, null); + } catch (IOException ex) { + LOGGER.log(Level.FINER, "could not skip " + repo.getId(), ex); + } + } else { + mutex.writeAccess((Mutex.Action) () -> { + try { + boolean index = indexer.loadIndexingContext(repo); + if (skipUnIndexed && index) { + if (!RepositoryPreferences.isIndexRepositories()) { + return null; + } + boolean spawned = indexer.spawnIndexLoadedRepo(repo); + if (spawned) { + actionSkip.run(repo, null); + } + return null; + } + IndexingContext context = indexer.getIndexingContexts().get(repo.getId()); + if (context == null) { + if (skipUnIndexed) { + actionSkip.run(repo, null); + } + return null; + } + action.run(repo, context); + } catch (IOException x) { + LOGGER.log(Level.INFO, "could not process " + repo.getId(), x); + } + return null; + }); + } + } + } + + @CheckForNull IteratorSearchResponse repeatedPagedSearch(Query q, IndexingContext context, int count) throws IOException { + return repeatedPagedSearch(q, List.of(context), count); + } + + @CheckForNull IteratorSearchResponse repeatedPagedSearch(Query q, List contexts, int count) throws IOException { + IteratorSearchRequest isr = new IteratorSearchRequest(q, contexts, new NoJavadocSourceFilter()); + if (count > 0) { + isr.setCount(count); + } + + int MAX_MAX_CLAUSE = 1<<11; // conservative maximum for too general queries, like "c:*class*" + + if (q instanceof BooleanQuery booleanQuery) { + List list = booleanQuery.clauses(); + if (list.size() == 1) { + Query q1 = list.get(0).getQuery(); + if (q1 instanceof PrefixQuery pq && "u".equals(pq.getPrefix().field())) { + // increase for queries like "+u:org.netbeans.modules|*" to succeed + MAX_MAX_CLAUSE = 1<<16; + } else if (q1 instanceof TermQuery tq && "p".equals(tq.getTerm().field())) { + // +p:nbm also produces several thousand hits + MAX_MAX_CLAUSE = 1<<16; + } + } + } + + int oldMax = IndexSearcher.getMaxClauseCount(); + try { + int max = oldMax; + while (true) { + IteratorSearchResponse response; + try { + IndexSearcher.setMaxClauseCount(max); + response = indexer.getSearcher().searchIteratorPaged(isr, contexts); + LOGGER.log(Level.FINE, "passed on {0} clauses processing {1} with {2} hits", new Object[] {max, q, response.getTotalHitsCount()}); + return response; + } catch (IndexSearcher.TooManyClauses exc) { + LOGGER.log(Level.FINE, "TooManyClauses on {0} clauses processing {1}", new Object[] {max, q}); + max *= 2; + if (max > MAX_MAX_CLAUSE) { + LOGGER.log(Level.WARNING, "Encountered more than {0} clauses processing {1}", new Object[] {MAX_MAX_CLAUSE, q}); + return null; + } + } + } + } finally { + IndexSearcher.setMaxClauseCount(oldMax); + } + } + + private Query constructQuery(Field f, String qs) { + return indexer.getIndexer().constructQuery(f, new StringSearchExpression(qs)); + } + + private static Query setBooleanRewrite(final Query q) { + if (q instanceof MultiTermQuery multiTermQuery) { + multiTermQuery.setRewriteMethod(MultiTermQuery.CONSTANT_SCORE_REWRITE); + } else if (q instanceof BooleanQuery booleanQuery) { + for (BooleanClause c : booleanQuery.clauses()) { + setBooleanRewrite(c.getQuery()); + } + } + return q; + } + + private static String toNexusField(String field) { + if (field != null) switch (field) { + case QueryField.FIELD_ARTIFACTID: return ArtifactInfo.ARTIFACT_ID; + case QueryField.FIELD_GROUPID: return ArtifactInfo.GROUP_ID; + case QueryField.FIELD_VERSION: return ArtifactInfo.VERSION; + case QueryField.FIELD_CLASSES: return ArtifactInfo.NAMES; + case QueryField.FIELD_NAME: return ArtifactInfo.NAME; + case QueryField.FIELD_DESCRIPTION: return ArtifactInfo.DESCRIPTION; + case QueryField.FIELD_PACKAGING: return ArtifactInfo.PACKAGING; + } + return field; + } + + static List convertToNBVersionInfo(Collection artifactInfos) { + List bVersionInfos = new ArrayList<>(); + for (ArtifactInfo ai : artifactInfos) { + NBVersionInfo nbvi = convertToNBVersionInfo(ai); + if (nbvi != null) { + bVersionInfos.add(nbvi); + } + } + return bVersionInfos; + } + + static NBVersionInfo convertToNBVersionInfo(ArtifactInfo ai) { + if ("javadoc".equals(ai.getClassifier()) || "sources".equals(ai.getClassifier())) { //NOI18N + // we don't want javadoc and sources shown anywhere, we use the getJavadocExists(), getSourceExists() methods. + return null; + } + // fextension != packaging - e.g a pom could be packaging "bundle" but from type/extension "jar" + NBVersionInfo nbvi = new NBVersionInfo(ai.getRepository(), ai.getGroupId(), ai.getArtifactId(), + ai.getVersion(), ai.getFileExtension(), ai.getPackaging(), ai.getName(), ai.getDescription(), ai.getClassifier()); + /*Javadoc & Sources*/ + nbvi.setJavadocExists(ai.getJavadocExists() == ArtifactAvailability.PRESENT); + nbvi.setSourcesExists(ai.getSourcesExists() == ArtifactAvailability.PRESENT); + nbvi.setSignatureExists(ai.getSignatureExists() == ArtifactAvailability.PRESENT); +// nbvi.setSha(ai.sha1); + nbvi.setLastModified(ai.getLastModified()); + nbvi.setSize(ai.getSize()); + nbvi.setLuceneScore(ai.getLuceneScore()); + return nbvi; + } + + @Override + public List getLoaded(final List repos) { + final List toRet = new ArrayList<>(repos.size()); + for (final RepositoryInfo repo : repos) { + Path loc = NexusRepositoryIndexManager.getIndexDirectory(repo); // index folder + if (NexusRepositoryIndexManager.indexExists(loc)) { + toRet.add(repo); + } + } + return toRet; + } + + synchronized SmoSearchBackend getSMO() { + if (smo == null) { + smo = SmoSearchBackendFactory.create( + SmoSearchBackendFactory.DEFAULT_BACKEND_ID, + SmoSearchBackendFactory.DEFAULT_REPOSITORY_ID, + SmoSearchBackendFactory.DEFAULT_SMO_URI, + new Java11HttpClientTransport(SMORequestResult.REQUEST_TIMEOUT) + ); + } + return smo; + } + + private interface RepoAction { + void run(RepositoryInfo repo, IndexingContext context) throws IOException; + } + + private record SkippedAction(ResultImpl result) implements RepoAction { + @Override + public void run(RepositoryInfo repo, IndexingContext context) throws IOException { + //indexing context is always null here.. + result.addSkipped(repo); + } + } + + private static class NoJavadocSourceFilter implements ArtifactInfoFilter { + @Override + public boolean accepts(IndexingContext ctx, ArtifactInfo ai) { + return !("javadoc".equals(ai.getClassifier()) || "sources".equals(ai.getClassifier())); + } + } + +} diff --git a/java/maven.indexer/src/org/netbeans/modules/maven/indexer/OnStop.java b/java/maven.indexer/src/org/netbeans/modules/maven/indexer/OnStop.java index 3004e3cc79fe..5deb9b8bea53 100644 --- a/java/maven.indexer/src/org/netbeans/modules/maven/indexer/OnStop.java +++ b/java/maven.indexer/src/org/netbeans/modules/maven/indexer/OnStop.java @@ -46,7 +46,7 @@ public void run() { } } for (RepositoryIndexerImplementation rii : Lookup.getDefault().lookupAll(RepositoryIndexerImplementation.class)) { - if (rii instanceof NexusRepositoryIndexerImpl impl) { + if (rii instanceof NexusRepositoryIndexManager impl) { impl.shutdownAll(); } } diff --git a/java/maven.indexer/src/org/netbeans/modules/maven/indexer/RemoteIndexTransferListener.java b/java/maven.indexer/src/org/netbeans/modules/maven/indexer/RemoteIndexTransferListener.java index 97690f8fd30a..72c34540eed1 100644 --- a/java/maven.indexer/src/org/netbeans/modules/maven/indexer/RemoteIndexTransferListener.java +++ b/java/maven.indexer/src/org/netbeans/modules/maven/indexer/RemoteIndexTransferListener.java @@ -38,7 +38,7 @@ import org.openide.util.Cancellable; import org.openide.util.NbBundle.Messages; -public class RemoteIndexTransferListener implements TransferListener, Cancellable { +public class RemoteIndexTransferListener implements TransferListener, Cancellable, AutoCloseable { private static final Logger LOG = Logger.getLogger(RemoteIndexTransferListener.class.getName()); @@ -167,7 +167,8 @@ void unpackingProgress(String label) { handle.progress(label); } - void close() { + @Override + public void close() { handle.finish(); } diff --git a/java/maven.indexer/src/org/netbeans/modules/maven/indexer/api/RepositoryIndexer.java b/java/maven.indexer/src/org/netbeans/modules/maven/indexer/api/RepositoryIndexer.java index bb20dde1ee2c..d3d0f41805f0 100644 --- a/java/maven.indexer/src/org/netbeans/modules/maven/indexer/api/RepositoryIndexer.java +++ b/java/maven.indexer/src/org/netbeans/modules/maven/indexer/api/RepositoryIndexer.java @@ -23,7 +23,7 @@ import java.util.Collection; import org.apache.maven.artifact.Artifact; -import org.netbeans.modules.maven.indexer.NexusRepositoryIndexerImpl; +import org.netbeans.modules.maven.indexer.NexusRepositoryIndexManager; import org.netbeans.modules.maven.indexer.spi.impl.RepositoryIndexerImplementation; import org.openide.util.Lookup; import org.netbeans.modules.maven.indexer.spi.RepositoryIndexQueryProvider; @@ -71,7 +71,7 @@ static RepositoryIndexerImplementation findImplementation(RepositoryInfo repo) { Lookup l = Lookup.getDefault(); Collection queryProviders = l.lookupAll(RepositoryIndexQueryProvider.class); for (RepositoryIndexQueryProvider queryProvider : queryProviders) { - if(!(queryProvider instanceof NexusRepositoryIndexerImpl) && queryProvider.handlesRepository(repo)) { + if(!(queryProvider instanceof NexusRepositoryIndexManager) && queryProvider.handlesRepository(repo)) { // skip if return null; } diff --git a/java/maven.indexer/src/org/netbeans/modules/maven/indexer/spi/impl/RepositoryIndexerImplementation.java b/java/maven.indexer/src/org/netbeans/modules/maven/indexer/spi/impl/RepositoryIndexerImplementation.java index 9f625d843ed2..ccd7cdec242e 100644 --- a/java/maven.indexer/src/org/netbeans/modules/maven/indexer/spi/impl/RepositoryIndexerImplementation.java +++ b/java/maven.indexer/src/org/netbeans/modules/maven/indexer/spi/impl/RepositoryIndexerImplementation.java @@ -27,9 +27,8 @@ * * Used internally. * - * Implementation of repository indexer (repository manager). Apart from basic - * indexing features also serves as provider of various index queries. - * There is one implementation based on apache indexer + * Implementation of repository indexer (repository manager). + * There is one implementation based on apache indexer. * * @author Milos Kleint */ diff --git a/java/maven.indexer/test/unit/src/org/netbeans/modules/maven/indexer/ClassDependencyIndexCreatorTest.java b/java/maven.indexer/test/unit/src/org/netbeans/modules/maven/indexer/ClassDependencyIndexCreatorTest.java index 83c3f102e99b..71079fd9203a 100644 --- a/java/maven.indexer/test/unit/src/org/netbeans/modules/maven/indexer/ClassDependencyIndexCreatorTest.java +++ b/java/maven.indexer/test/unit/src/org/netbeans/modules/maven/indexer/ClassDependencyIndexCreatorTest.java @@ -30,6 +30,8 @@ import org.netbeans.modules.maven.indexer.api.NBVersionInfo; import org.netbeans.modules.maven.indexer.api.RepositoryInfo; import org.netbeans.modules.maven.indexer.api.RepositoryQueries.ClassUsage; +import org.netbeans.modules.maven.indexer.spi.ClassUsageQuery; +import org.netbeans.modules.maven.indexer.spi.ClassesQuery; import org.openide.filesystems.FileUtil; import org.openide.util.test.JarBuilder; import org.openide.util.test.TestFileUtils; @@ -87,21 +89,22 @@ public void testFindClassUsages() throws Exception { nrii.indexRepo(info); List repo = List.of(info); - List list = nrii.findClassUsages("mod1.API", repo).getResults(); + ClassUsageQuery query = nrii.getClassUsageQuery(); + List list = query.findClassUsages("mod1.API", repo).getResults(); assertEquals("[test:mod2:0:test[mod2.Client, mod2.OtherClient], test:mod3:0:test[mod3.Client]]", list.toString()); - list = nrii.findClassUsages("mod1.Util", repo).getResults(); + list = query.findClassUsages("mod1.Util", repo).getResults(); assertEquals("[test:mod4:0:test[mod4.Install]]", list.toString()); assertEquals("jar", list.get(0).getArtifact().getType()); - list = nrii.findClassUsages("mod1.Stuff", repo).getResults(); + list = query.findClassUsages("mod1.Stuff", repo).getResults(); assertEquals("[test:mod5:0:test[mod5.Install]]", list.toString()); assertEquals("jar", list.get(0).getArtifact().getType()); - assertEquals("[]", nrii.findClassUsages("java.lang.Object", repo).getResults().toString()); - assertEquals("[test:mod2:0:test[mod2.Outer]]", nrii.findClassUsages("mod1.Outer", repo).getResults().toString()); - assertEquals("[test:mod2:0:test[mod2.Outer]]", nrii.findClassUsages("mod1.Outer$Inner", repo).getResults().toString()); - assertEquals("[]", nrii.findClassUsages("mod1.Outer$Unused", repo).getResults().toString()); + assertEquals("[]", query.findClassUsages("java.lang.Object", repo).getResults().toString()); + assertEquals("[test:mod2:0:test[mod2.Outer]]", query.findClassUsages("mod1.Outer", repo).getResults().toString()); + assertEquals("[test:mod2:0:test[mod2.Outer]]", query.findClassUsages("mod1.Outer$Inner", repo).getResults().toString()); + assertEquals("[]", query.findClassUsages("mod1.Outer$Unused", repo).getResults().toString()); // XXX InnerClass attribute will produce spurious references to outer classes even when just an inner is used } @@ -129,18 +132,19 @@ public void testFindVersionsByClass() throws Exception { nrii.indexRepo(info); List repo = List.of(info); + ClassesQuery query = nrii.getClassesQuery(); // single version - List list = nrii.findVersionsByClass("mod1.API", repo).getResults(); + List list = query.findVersionsByClass("mod1.API", repo).getResults(); assertEquals("[test:mod1:42:test]", list.toString()); - list = nrii.findVersionsByClass("mod1.Util", repo).getResults(); + list = query.findVersionsByClass("mod1.Util", repo).getResults(); assertEquals("[test:mod1:42:test]", list.toString()); // two versions - list = nrii.findVersionsByClass("mod2.API2", repo).getResults(); + list = query.findVersionsByClass("mod2.API2", repo).getResults(); assertEquals("[test:mod2:43:test, test:mod2:42:test]", list.toString()); - list = nrii.findVersionsByClass("mod2.Util2", repo).getResults(); + list = query.findVersionsByClass("mod2.Util2", repo).getResults(); assertEquals("[test:mod2:43:test]", list.toString()); } diff --git a/java/maven.indexer/test/unit/src/org/netbeans/modules/maven/indexer/NexusRepositoryIndexerImplTest.java b/java/maven.indexer/test/unit/src/org/netbeans/modules/maven/indexer/NexusRepositoryIndexerImplTest.java index 536091b3219e..1d0d14efe15d 100644 --- a/java/maven.indexer/test/unit/src/org/netbeans/modules/maven/indexer/NexusRepositoryIndexerImplTest.java +++ b/java/maven.indexer/test/unit/src/org/netbeans/modules/maven/indexer/NexusRepositoryIndexerImplTest.java @@ -55,7 +55,7 @@ public void testFind() throws Exception { qf.setValue("stuff"); qf.setOccur(QueryField.OCCUR_MUST); qf.setMatch(QueryField.MATCH_EXACT); - assertEquals("[test:plugin:0:test]", nrii.find(List.of(qf), List.of(info)).getResults().toString()); + assertEquals("[test:plugin:0:test]", nrii.getGenericFindQuery().find(List.of(qf), List.of(info)).getResults().toString()); } public void testLastUpdated() throws Exception { // #197670 @@ -65,7 +65,7 @@ public void testLastUpdated() throws Exception { // #197670 install(empty, "test", "art", "0", "pom.lastUpdated"); install(empty, "test", "art", "0", "jar.lastUpdated"); nrii.indexRepo(info); - List versions = nrii.getVersions("test", "art", List.of(info)).getResults(); + List versions = nrii.getBaseQueries().getVersions("test", "art", List.of(info)).getResults(); assertEquals(1, versions.size()); NBVersionInfo v = versions.get(0); assertEquals("test:art:0:test", v.toString()); diff --git a/java/maven.indexer/test/unit/src/org/netbeans/modules/maven/indexer/NexusTestBase.java b/java/maven.indexer/test/unit/src/org/netbeans/modules/maven/indexer/NexusTestBase.java index ea3fada3252a..3ff788ef7ef4 100644 --- a/java/maven.indexer/test/unit/src/org/netbeans/modules/maven/indexer/NexusTestBase.java +++ b/java/maven.indexer/test/unit/src/org/netbeans/modules/maven/indexer/NexusTestBase.java @@ -44,7 +44,7 @@ protected NexusTestBase(String n) { private MavenEmbedder embedder; private ArtifactInstaller artifactInstaller; protected RepositoryInfo info; - protected NexusRepositoryIndexerImpl nrii; + protected NexusRepositoryIndexManager nrii; @Override protected void setUp() throws Exception { clearWorkDir(); @@ -57,7 +57,7 @@ protected NexusTestBase(String n) { artifactInstaller = embedder.lookupComponent(ArtifactInstaller.class); info = new RepositoryInfo("test", "Test", repo.getAbsolutePath(), null); RepositoryPreferences.getInstance().addOrModifyRepositoryInfo(info); - nrii = Lookup.getDefault().lookup(NexusRepositoryIndexerImpl.class); + nrii = Lookup.getDefault().lookup(NexusRepositoryIndexManager.class); } @Override protected Level logLevel() {