[REFACTORING] Project Scanning Performance

[stephanj]

The ProjectScannerService recursively scans project files but doesn't optimize for large projects.

Example Problem:

private void walkThroughDirectory(@NotNull VirtualFile directory,
                               @NotNull ProjectFileIndex fileIndex,
                               @NotNull StringBuilder fullContent,
                               @NotNull ScanContentResult scanContentResult) {
    VfsUtilCore.visitChildrenRecursively(directory, new VirtualFileVisitor<Void>() {
        @Override
        public boolean visitFile(@NotNull VirtualFile file) {
            // Process each file
            if (!file.isDirectory()) {
                readFileContent(file, fullContent, scanContentResult);
            }
            return true;
        }
    });
}

Recommendation:
Implement early termination based on token limits and more efficient traversal:

private void walkThroughDirectory(@NotNull VirtualFile directory,
                                @NotNull ProjectFileIndex fileIndex,
                                @NotNull StringBuilder fullContent,
                                @NotNull ScanContentResult scanContentResult,
                                int maxTokenLimit) {
    
    AtomicInteger tokenCount = new AtomicInteger(0);
    Queue<VirtualFile> directoriesToProcess = new LinkedList<>();
    directoriesToProcess.add(directory);
    
    while (!directoriesToProcess.isEmpty() && tokenCount.get() < maxTokenLimit) {
        VirtualFile currentDir = directoriesToProcess.poll();
        VirtualFile[] children = currentDir.getChildren();
        
        // Process directories first (breadth-first)
        for (VirtualFile child : children) {
            if (child.isDirectory() && !shouldExcludeDirectory(child)) {
                directoriesToProcess.add(child);
            }
        }
        
        // Process files
        for (VirtualFile child : children) {
            if (!child.isDirectory() && tokenCount.get() < maxTokenLimit) {
                if (shouldIncludeFile(child)) {
                    int fileTokens = readFileContent(child, fullContent, scanContentResult);
                    tokenCount.addAndGet(fileTokens);
                    
                    if (tokenCount.get() >= maxTokenLimit) {
                        break; // Stop if we've reached the token limit
                    }
                }
            }
        }
    }
}