Merge pull request #1216 from ahoppen/background-indexing

Implement initial background indexing of a project

.gitignore

@@ -2,6 +2,7 @@
 default.profraw
 Package.resolved
 /.build
+/.index-build
 /Packages
 /*.xcodeproj
 /*.sublime-project

Package.swift

@@ -173,6 +173,8 @@ let package = Package(
     .target(
       name: "SemanticIndex",
       dependencies: [
+        "CAtomics",
+        "LanguageServerProtocol",
         "LSPLogging",
         "SKCore",
         .product(name: "IndexStoreDB", package: "indexstore-db"),

Sources/LSPLogging/Logging.swift

@@ -13,7 +13,7 @@
 /// Which log level to use (from https://developer.apple.com/wwdc20/10168?time=604)
 ///  - Debug: Useful only during debugging (only logged during debugging)
 ///  - Info: Helpful but not essential for troubleshooting (not persisted, logged to memory)
-///  - Notice/log (Default): Essential for troubleshooting
+///  - Notice/log/default: Essential for troubleshooting
 ///  - Error: Error seen during execution
 ///    - Used eg. if the user sends an erroneous request or if a request fails
 ///  - Fault: Bug in program

Sources/LSPLogging/NonDarwinLogging.swift

@@ -330,7 +330,7 @@ public struct NonDarwinLogger: Sendable {
     log(level: .info, message)
   }
 
-  /// Log a message at the `log` level.
+  /// Log a message at the `default` level.
   public func log(_ message: NonDarwinLogMessage) {
     log(level: .default, message)
   }

Sources/SKCore/BuildSystemManager.swift

@@ -159,7 +159,8 @@ extension BuildSystemManager {
   /// references to that C file in the build settings by the header file.
   public func buildSettingsInferredFromMainFile(
     for document: DocumentURI,
-    language: Language
+    language: Language,
+    logBuildSettings: Bool = true
   ) async -> FileBuildSettings? {
     let mainFile = await mainFile(for: document, language: language)
     guard var settings = await buildSettings(for: mainFile, language: language) else {
@@ -170,7 +171,9 @@ extension BuildSystemManager {
       // to reference `document` instead of `mainFile`.
       settings = settings.patching(newFile: document.pseudoPath, originalFile: mainFile.pseudoPath)
     }
-    await BuildSettingsLogger.shared.log(settings: settings, for: document)
+    if logBuildSettings {
+      await BuildSettingsLogger.shared.log(settings: settings, for: document)
+    }
     return settings
   }
 
@@ -349,16 +352,24 @@ extension BuildSystemManager {
 // MARK: - Build settings logger
 
 /// Shared logger that only logs build settings for a file once unless they change
-fileprivate actor BuildSettingsLogger {
-  static let shared = BuildSettingsLogger()
+public actor BuildSettingsLogger {
+  public static let shared = BuildSettingsLogger()
 
   private var loggedSettings: [DocumentURI: FileBuildSettings] = [:]
 
-  func log(settings: FileBuildSettings, for uri: DocumentURI) {
+  public func log(level: LogLevel = .default, settings: FileBuildSettings, for uri: DocumentURI) {
     guard loggedSettings[uri] != settings else {
       return
     }
     loggedSettings[uri] = settings
+    Self.log(level: level, settings: settings, for: uri)
+  }
+
+  /// Log the given build settings.
+  ///
+  /// In contrast to the instance method `log`, this will always log the build settings. The instance method only logs
+  /// the build settings if they have changed.
+  public static func log(level: LogLevel = .default, settings: FileBuildSettings, for uri: DocumentURI) {
     let log = """
       Compiler Arguments:
       \(settings.compilerArguments.joined(separator: "\n"))
@@ -370,6 +381,7 @@ fileprivate actor BuildSettingsLogger {
     let chunks = splitLongMultilineMessage(message: log)
     for (index, chunk) in chunks.enumerated() {
       logger.log(
+        level: level,
         """
         Build settings for \(uri.forLogging) (\(index + 1)/\(chunks.count))
         \(chunk)

Sources/SKSupport/CMakeLists.txt

@@ -14,6 +14,7 @@ add_library(SKSupport STATIC
   Process+WaitUntilExitWithCancellation.swift
   Random.swift
   Result.swift
+  SwitchableProcessResultExitStatus.swift
   ThreadSafeBox.swift
   WorkspaceType.swift
 )

Sources/SKSupport/SwitchableProcessResultExitStatus.swift

@@ -0,0 +1,45 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2024 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+// We need to import all of TSCBasic because otherwise we can't refer to Process.ExitStatus (rdar://127577691)
+import struct TSCBasic.ProcessResult
+
+/// Same as `ProcessResult.ExitStatus` in tools-support-core but has the same cases on all platforms and is thus easier
+/// to switch over
+public enum SwitchableProcessResultExitStatus {
+  /// The process was terminated normally with a exit code.
+  case terminated(code: Int32)
+  /// The process was terminated abnormally.
+  case abnormal(exception: UInt32)
+  /// The process was terminated due to a signal.
+  case signalled(signal: Int32)
+}
+
+extension ProcessResult.ExitStatus {
+  public var exhaustivelySwitchable: SwitchableProcessResultExitStatus {
+    #if os(Windows)
+    switch self {
+    case .terminated(let code):
+      return .terminated(code: code)
+    case .abnormal(let exception):
+      return .abnormal(exception: exception)
+    }
+    #else
+    switch self {
+    case .terminated(let code):
+      return .terminated(code: code)
+    case .signalled(let signal):
+      return .signalled(signal: signal)
+    }
+    #endif
+  }
+}

Sources/SKTestSupport/SwiftPMTestProject.swift

@@ -43,6 +43,7 @@ public class SwiftPMTestProject: MultiFileTestProject {
     build: Bool = false,
     allowBuildFailure: Bool = false,
     serverOptions: SourceKitLSPServer.Options = .testDefault,
+    pollIndex: Bool = true,
     usePullDiagnostics: Bool = true,
     testName: String = #function
   ) async throws {
@@ -77,8 +78,10 @@ public class SwiftPMTestProject: MultiFileTestProject {
         try await Self.build(at: self.scratchDirectory)
       }
     }
-    // Wait for the indexstore-db to finish indexing
-    _ = try await testClient.send(PollIndexRequest())
+    if pollIndex {
+      // Wait for the indexstore-db to finish indexing
+      _ = try await testClient.send(PollIndexRequest())
+    }
   }
 
   /// Build a SwiftPM package package manifest is located in the directory at `path`.

Sources/SKTestSupport/WrappedSemaphore.swift

@@ -0,0 +1,36 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2024 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import Dispatch
+
+/// Wrapper around `DispatchSemaphore` so that Swift Concurrency doesn't complain about the usage of semaphores in the
+/// tests.
+///
+/// This should only be used for tests that test priority escalation and thus cannot await a `Task` (which would cause
+/// priority elevations).
+public struct WrappedSemaphore {
+  let semaphore = DispatchSemaphore(value: 0)
+
+  public init() {}
+
+  public func signal(value: Int = 1) {
+    for _ in 0..<value {
+      semaphore.signal()
+    }
+  }
+
+  public func wait(value: Int = 1) {
+    for _ in 0..<value {
+      semaphore.wait()
+    }
+  }
+}

Sources/SemanticIndex/CMakeLists.txt

@@ -1,6 +1,8 @@
 
 add_library(SemanticIndex STATIC
   CheckedIndex.swift
+  SemanticIndexManager.swift
+  UpdateIndexStoreTaskDescription.swift
 )
 set_target_properties(SemanticIndex PROPERTIES
   INTERFACE_INCLUDE_DIRECTORIES ${CMAKE_Swift_MODULE_DIRECTORY})

Sources/SemanticIndex/SemanticIndexManager.swift

@@ -0,0 +1,167 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2024 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import Foundation
+import LSPLogging
+import LanguageServerProtocol
+import SKCore
+
+/// Describes the state of indexing for a single source file
+private enum FileIndexStatus {
+  /// The index is up-to-date.
+  case upToDate
+  /// The file is being indexed by the given task.
+  case inProgress(Task<Void, Never>)
+}
+
+/// Schedules index tasks and keeps track of the index status of files.
+public final actor SemanticIndexManager {
+  /// The underlying index. This is used to check if the index of a file is already up-to-date, in which case it doesn't
+  /// need to be indexed again.
+  private let index: CheckedIndex
+
+  /// The build system manager that is used to get compiler arguments for a file.
+  private let buildSystemManager: BuildSystemManager
+
+  /// The index status of the source files that the `SemanticIndexManager` knows about.
+  ///
+  /// Files that have never been indexed are not in this dictionary.
+  private var indexStatus: [DocumentURI: FileIndexStatus] = [:]
+
+  /// The `TaskScheduler` that manages the scheduling of index tasks. This is shared among all `SemanticIndexManager`s
+  /// in the process, to ensure that we don't schedule more index operations than processor cores from multiple
+  /// workspaces.
+  private let indexTaskScheduler: TaskScheduler<UpdateIndexStoreTaskDescription>
+
+  /// Callback that is called when an index task has finished.
+  ///
+  /// Currently only used for testing.
+  private let indexTaskDidFinish: (@Sendable (UpdateIndexStoreTaskDescription) -> Void)?
+
+  // MARK: - Public API
+
+  public init(
+    index: UncheckedIndex,
+    buildSystemManager: BuildSystemManager,
+    indexTaskScheduler: TaskScheduler<UpdateIndexStoreTaskDescription>,
+    indexTaskDidFinish: (@Sendable (UpdateIndexStoreTaskDescription) -> Void)?
+  ) {
+    self.index = index.checked(for: .modifiedFiles)
+    self.buildSystemManager = buildSystemManager
+    self.indexTaskScheduler = indexTaskScheduler
+    self.indexTaskDidFinish = indexTaskDidFinish
+  }
+
+  /// Schedules a task to index all files in `files` that don't already have an up-to-date index.
+  /// Returns immediately after scheduling that task.
+  ///
+  /// Indexing is being performed with a low priority.
+  public func scheduleBackgroundIndex(files: some Collection<DocumentURI>) {
+    self.index(files: files, priority: .low)
+  }
+
+  /// Wait for all in-progress index tasks to finish.
+  public func waitForUpToDateIndex() async {
+    logger.info("Waiting for up-to-date index")
+    await withTaskGroup(of: Void.self) { taskGroup in
+      for (_, status) in indexStatus {
+        switch status {
+        case .inProgress(let task):
+          taskGroup.addTask {
+            await task.value
+          }
+        case .upToDate:
+          break
+        }
+      }
+      await taskGroup.waitForAll()
+    }
+    index.pollForUnitChangesAndWait()
+    logger.debug("Done waiting for up-to-date index")
+  }
+
+  /// Ensure that the index for the given files is up-to-date.
+  ///
+  /// This tries to produce an up-to-date index for the given files as quickly as possible. To achieve this, it might
+  /// suspend previous target-wide index tasks in favor of index tasks that index a fewer files.
+  public func waitForUpToDateIndex(for uris: some Collection<DocumentURI>) async {
+    logger.info(
+      "Waiting for up-to-date index for \(uris.map { $0.fileURL?.lastPathComponent ?? $0.stringValue }.joined(separator: ", "))"
+    )
+    let filesWithOutOfDateIndex = uris.filter { uri in
+      switch indexStatus[uri] {
+      case .inProgress, nil: return true
+      case .upToDate: return false
+      }
+    }
+    // Create a new index task for the files that aren't up-to-date. The newly scheduled index tasks will
+    // - Wait for the existing index operations to finish if they have the same number of files.
+    // - Reschedule the background index task in favor of an index task with fewer source files.
+    await self.index(files: filesWithOutOfDateIndex, priority: nil).value
+    index.pollForUnitChangesAndWait()
+    logger.debug("Done waiting for up-to-date index")
+  }
+
+  // MARK: - Helper functions
+
+  /// Index the given set of files at the given priority.
+  ///
+  /// The returned task finishes when all files are indexed.
+  @discardableResult
+  private func index(files: some Collection<DocumentURI>, priority: TaskPriority?) -> Task<Void, Never> {
+    let outOfDateFiles = files.filter {
+      if case .upToDate = indexStatus[$0] {
+        return false
+      }
+      return true
+    }
+
+    var indexTasks: [Task<Void, Never>] = []
+
+    // TODO (indexing): Group index operations by target when we support background preparation.
+    for files in outOfDateFiles.partition(intoNumberOfBatches: ProcessInfo.processInfo.processorCount * 5) {
+      let indexTask = Task(priority: priority) {
+        await self.indexTaskScheduler.schedule(
+          priority: priority,
+          UpdateIndexStoreTaskDescription(
+            filesToIndex: Set(files),
+            buildSystemManager: self.buildSystemManager,
+            index: self.index,
+            didFinishCallback: { [weak self] taskDescription in
+              self?.indexTaskDidFinish?(taskDescription)
+            }
+          )
+        ).value
+        for file in files {
+          indexStatus[file] = .upToDate
+        }
+      }
+      indexTasks.append(indexTask)
+
+      for file in files {
+        indexStatus[file] = .inProgress(indexTask)
+      }
+    }
+    let indexTasksImmutable = indexTasks
+
+    return Task(priority: priority) {
+      await withTaskGroup(of: Void.self) { taskGroup in
+        for indexTask in indexTasksImmutable {
+          taskGroup.addTask(priority: priority) {
+            await indexTask.value
+          }
+        }
+        await taskGroup.waitForAll()
+      }
+    }
+  }
+}

Sources/SemanticIndex/UpdateIndexStoreTaskDescription.swift

@@ -0,0 +1,311 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2024 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import CAtomics
+import Foundation
+import LSPLogging
+import LanguageServerProtocol
+import SKCore
+import SKSupport
+
+import struct TSCBasic.AbsolutePath
+import class TSCBasic.Process
+
+private var updateIndexStoreIDForLogging = AtomicUInt32(initialValue: 1)
+
+/// Describes a task to index a set of source files.
+///
+/// This task description can be scheduled in a `TaskScheduler`.
+public struct UpdateIndexStoreTaskDescription: TaskDescriptionProtocol {
+  public let id = updateIndexStoreIDForLogging.fetchAndIncrement()
+
+  /// The files that should be indexed.
+  private let filesToIndex: Set<DocumentURI>
+
+  /// The build system manager that is used to get the toolchain and build settings for the files to index.
+  private let buildSystemManager: BuildSystemManager
+
+  /// A reference to the underlying index store. Used to check if the index is already up-to-date for a file, in which
+  /// case we don't need to index it again.
+  private let index: CheckedIndex
+
+  /// A callback that is called when the index task finishes
+  private let didFinishCallback: @Sendable (UpdateIndexStoreTaskDescription) -> Void
+
+  /// The task is idempotent because indexing the same file twice produces the same result as indexing it once.
+  public var isIdempotent: Bool { true }
+
+  public var estimatedCPUCoreCount: Int { 1 }
+
+  public var description: String {
+    return self.redactedDescription
+  }
+
+  public var redactedDescription: String {
+    return "indexing-\(id)"
+  }
+
+  init(
+    filesToIndex: Set<DocumentURI>,
+    buildSystemManager: BuildSystemManager,
+    index: CheckedIndex,
+    didFinishCallback: @escaping @Sendable (UpdateIndexStoreTaskDescription) -> Void
+  ) {
+    self.filesToIndex = filesToIndex
+    self.buildSystemManager = buildSystemManager
+    self.index = index
+    self.didFinishCallback = didFinishCallback
+  }
+
+  public func execute() async {
+    defer {
+      didFinishCallback(self)
+    }
+    // Only use the last two digits of the indexing ID for the logging scope to avoid creating too many scopes.
+    // See comment in `withLoggingScope`.
+    // The last 2 digits should be sufficient to differentiate between multiple concurrently running indexing operation.
+    await withLoggingSubsystemAndScope(
+      subsystem: "org.swift.sourcekit-lsp.indexing",
+      scope: "update-indexstore-\(id % 100)"
+    ) {
+      let startDate = Date()
+
+      let filesToIndexDescription = filesToIndex.map { $0.fileURL?.lastPathComponent ?? $0.stringValue }
+        .joined(separator: ", ")
+      logger.log(
+        "Starting updating index store with priority \(Task.currentPriority.rawValue, privacy: .public): \(filesToIndexDescription)"
+      )
+      let filesToIndex = filesToIndex.sorted(by: { $0.stringValue < $1.stringValue })
+      // TODO (indexing): Once swiftc supports it, we should group files by target and index files within the same
+      // target together in one swiftc invocation.
+      for file in filesToIndex {
+        await updateIndexStoreForSingleFile(file)
+      }
+      logger.log(
+        "Finished updating index store in \(Date().timeIntervalSince(startDate) * 1000, privacy: .public)ms: \(filesToIndexDescription)"
+      )
+    }
+  }
+
+  public func dependencies(
+    to currentlyExecutingTasks: [UpdateIndexStoreTaskDescription]
+  ) -> [TaskDependencyAction<UpdateIndexStoreTaskDescription>] {
+    return currentlyExecutingTasks.compactMap { (other) -> TaskDependencyAction<UpdateIndexStoreTaskDescription>? in
+      guard !other.filesToIndex.intersection(filesToIndex).isEmpty else {
+        // Disjoint sets of files can be indexed concurrently.
+        return nil
+      }
+      if self.filesToIndex.count < other.filesToIndex.count {
+        // If there is an index operation with more files already running, suspend it.
+        // The most common use case for this is if we schedule an entire target to be indexed in the background and then
+        // need a single file indexed for use interaction. We should suspend the target-wide indexing and just index
+        // the current file to get index data for it ASAP.
+        return .cancelAndRescheduleDependency(other)
+      } else {
+        return .waitAndElevatePriorityOfDependency(other)
+      }
+    }
+  }
+
+  private func updateIndexStoreForSingleFile(_ uri: DocumentURI) async {
+    guard let url = uri.fileURL else {
+      // The URI is not a file, so there's nothing we can index.
+      return
+    }
+    guard !index.hasUpToDateUnit(for: url) else {
+      // We consider a file's index up-to-date if we have any up-to-date unit. Changing build settings does not
+      // invalidate the up-to-date status of the index.
+      return
+    }
+    guard let language = await buildSystemManager.defaultLanguage(for: uri) else {
+      logger.error("Not indexing \(uri.forLogging) because its language could not be determined")
+      return
+    }
+    let buildSettings = await buildSystemManager.buildSettingsInferredFromMainFile(
+      for: uri,
+      language: language,
+      logBuildSettings: false
+    )
+    guard let buildSettings else {
+      logger.error("Not indexing \(uri.forLogging) because it has no compiler arguments")
+      return
+    }
+    guard !buildSettings.isFallback else {
+      // Only index with real build settings. Indexing with fallback arguments could result in worse results than not
+      // indexing at all: If a file has been indexed with real build settings before, had a tiny modification made but
+      // we don't have any real build settings when it should get re-indexed. Then it's better to have the stale index
+      // from correct compiler arguments than no index at all.
+      logger.error("Not updating index store for \(uri.forLogging) because it has fallback compiler arguments")
+      return
+    }
+    guard let toolchain = await buildSystemManager.toolchain(for: uri, language) else {
+      logger.error(
+        "Not updating index store for \(uri.forLogging) because no toolchain could be determined for the document"
+      )
+      return
+    }
+    switch language {
+    case .swift:
+      do {
+        try await updateIndexStore(forSwiftFile: uri, buildSettings: buildSettings, toolchain: toolchain)
+      } catch {
+        logger.error("Updating index store for \(uri) failed: \(error.forLogging)")
+        BuildSettingsLogger.log(settings: buildSettings, for: uri)
+      }
+    case .c, .cpp, .objective_c, .objective_cpp:
+      // TODO (indexing): Support indexing of clang files, including headers.
+      break
+    default:
+      logger.error(
+        "Not updating index store for \(uri) because it is a language that is not supported by background indexing"
+      )
+    }
+  }
+
+  private func updateIndexStore(
+    forSwiftFile uri: DocumentURI,
+    buildSettings: FileBuildSettings,
+    toolchain: Toolchain
+  ) async throws {
+    let indexingArguments = adjustSwiftCompilerArgumentsForIndexStoreUpdate(
+      buildSettings.compilerArguments,
+      fileToIndex: uri
+    )
+
+    guard let swiftc = toolchain.swiftc else {
+      logger.error(
+        "Not updating index store for \(uri.forLogging) because toolchain \(toolchain.identifier) does not contain a Swift compiler"
+      )
+      return
+    }
+
+    let process =
+      if let workingDirectory = buildSettings.workingDirectory {
+        Process(
+          arguments: [swiftc.pathString] + indexingArguments,
+          workingDirectory: try AbsolutePath(validating: workingDirectory)
+        )
+      } else {
+        Process(arguments: [swiftc.pathString] + indexingArguments)
+      }
+    try process.launch()
+    let result = try await process.waitUntilExitSendingSigIntOnTaskCancellation()
+    switch result.exitStatus.exhaustivelySwitchable {
+    case .terminated(code: 0):
+      break
+    case .terminated(code: let code):
+      // This most likely happens if there are compilation errors in the source file. This is nothing to worry about.
+      let stdout = (try? String(bytes: result.output.get(), encoding: .utf8)) ?? "<no stderr>"
+      let stderr = (try? String(bytes: result.stderrOutput.get(), encoding: .utf8)) ?? "<no stderr>"
+      // Indexing will frequently fail if the source code is in an invalid state. Thus, log the failure at a low level.
+      logger.debug(
+        """
+        Updating index store for Swift file \(uri.forLogging) terminated with non-zero exit code \(code)
+        Stderr:
+        \(stderr)
+        Stdout:
+        \(stdout)
+        """
+      )
+      BuildSettingsLogger.log(level: .debug, settings: buildSettings, for: uri)
+    case .signalled(signal: let signal):
+      if !Task.isCancelled {
+        // The indexing job finished with a signal. Could be because the compiler crashed.
+        // Ignore signal exit codes if this task has been cancelled because the compiler exits with SIGINT if it gets
+        // interrupted.
+        logger.error("Updating index store for Swift file \(uri.forLogging) signaled \(signal)")
+        BuildSettingsLogger.log(level: .error, settings: buildSettings, for: uri)
+      }
+    case .abnormal(exception: let exception):
+      if !Task.isCancelled {
+        logger.error("Updating index store for Swift file \(uri.forLogging) exited abnormally \(exception)")
+        BuildSettingsLogger.log(level: .error, settings: buildSettings, for: uri)
+      }
+    }
+  }
+}
+
+/// Adjust compiler arguments that were created for building to compiler arguments that should be used for indexing.
+///
+/// This removes compiler arguments that produce output files and adds arguments to index the file.
+private func adjustSwiftCompilerArgumentsForIndexStoreUpdate(
+  _ compilerArguments: [String],
+  fileToIndex: DocumentURI
+) -> [String] {
+  let removeFlags: Set<String> = [
+    "-c",
+    "-disable-cmo",
+    "-emit-dependencies",
+    "-emit-module-interface",
+    "-emit-module",
+    "-emit-module",
+    "-emit-objc-header",
+    "-incremental",
+    "-no-color-diagnostics",
+    "-parseable-output",
+    "-save-temps",
+    "-serialize-diagnostics",
+    "-use-frontend-parseable-output",
+    "-validate-clang-modules-once",
+    "-whole-module-optimization",
+  ]
+
+  let removeArguments: Set<String> = [
+    "-clang-build-session-file",
+    "-emit-module-interface-path",
+    "-emit-module-path",
+    "-emit-objc-header-path",
+    "-emit-package-module-interface-path",
+    "-emit-private-module-interface-path",
+    "-num-threads",
+    "-o",
+    "-output-file-map",
+  ]
+
+  let removeFrontendFlags: Set<String> = [
+    "-experimental-skip-non-inlinable-function-bodies",
+    "-experimental-skip-all-function-bodies",
+  ]
+
+  var result: [String] = []
+  result.reserveCapacity(compilerArguments.count)
+  var iterator = compilerArguments.makeIterator()
+  while let argument = iterator.next() {
+    if removeFlags.contains(argument) {
+      continue
+    }
+    if removeArguments.contains(argument) {
+      _ = iterator.next()
+      continue
+    }
+    if argument == "-Xfrontend" {
+      if let nextArgument = iterator.next() {
+        if removeFrontendFlags.contains(nextArgument) {
+          continue
+        }
+        result += [argument, nextArgument]
+        continue
+      }
+      result.append(argument)
+    }
+    result.append(argument)
+  }
+  result += [
+    "-index-file",
+    "-index-file-path", fileToIndex.pseudoPath,
+    // batch mode is not compatible with -index-file
+    "-disable-batch-mode",
+    // Fake an output path so that we get a different unit file for every Swift file we background index
+    "-o", fileToIndex.pseudoPath + ".o",
+  ]
+  return result
+}

Sources/SourceKitLSP/SourceKitLSPServer.swift

@@ -449,6 +449,12 @@ public actor SourceKitLSPServer {
 
   let documentManager = DocumentManager()
 
+  /// The `TaskScheduler` that schedules all background indexing tasks.
+  ///
+  /// Shared process-wide to ensure the scheduled index operations across multiple workspaces don't exceed the maximum
+  /// number of processor cores that the user allocated to background indexing.
+  private let indexTaskScheduler: TaskScheduler<UpdateIndexStoreTaskDescription>
+
   private var packageLoadingWorkDoneProgress = WorkDoneProgressState(
     "SourceKitLSP.SourceKitLSPServer.reloadPackage",
     title: "SourceKit-LSP: Reloading Package"
@@ -519,6 +525,12 @@ public actor SourceKitLSPServer {
     self.onExit = onExit
 
     self.client = client
+    let processorCount = ProcessInfo.processInfo.processorCount
+    let lowPriorityCores = options.indexOptions.maxCoresPercentageToUseForBackgroundIndexing * Double(processorCount)
+    self.indexTaskScheduler = TaskScheduler(maxConcurrentTasksByPriority: [
+      (TaskPriority.medium, processorCount),
+      (TaskPriority.low, max(Int(lowPriorityCores), 1)),
+    ])
   }
 
   /// Search through all the parent directories of `uri` and check if any of these directories contain a workspace
@@ -1152,6 +1164,7 @@ extension SourceKitLSPServer {
       options: options,
       compilationDatabaseSearchPaths: self.options.compilationDatabaseSearchPaths,
       indexOptions: self.options.indexOptions,
+      indexTaskScheduler: indexTaskScheduler,
       reloadPackageStatusCallback: { [weak self] status in
         guard let self else { return }
         guard capabilityRegistry.clientCapabilities.window?.workDoneProgress ?? false else {
@@ -1220,7 +1233,8 @@ extension SourceKitLSPServer {
           options: self.options,
           underlyingBuildSystem: nil,
           index: nil,
-          indexDelegate: nil
+          indexDelegate: nil,
+          indexTaskScheduler: self.indexTaskScheduler
         )
 
         self.workspacesAndIsImplicit.append((workspace: workspace, isImplicit: false))
@@ -2414,7 +2428,8 @@ extension SourceKitLSPServer {
 
   func pollIndex(_ req: PollIndexRequest) async throws -> VoidResponse {
     for workspace in workspaces {
-      workspace.uncheckedIndex?.underlyingIndexStoreDB.pollForUnitChangesAndWait()
+      await workspace.semanticIndexManager?.waitForUpToDateIndex()
+      workspace.index(checkedFor: .deletedFiles)?.pollForUnitChangesAndWait()
     }
     return VoidResponse()
   }

Sources/SourceKitLSP/Workspace.swift

@@ -74,6 +74,12 @@ public final class Workspace {
   /// Language service for an open document, if available.
   var documentService: [DocumentURI: LanguageService] = [:]
 
+  /// The `SemanticIndexManager` that keeps track of whose file's index is up-to-date in the workspace and schedules
+  /// indexing and preparation tasks for files with out-of-date index.
+  ///
+  /// `nil` if background indexing is not enabled.
+  let semanticIndexManager: SemanticIndexManager?
+
   public init(
     documentManager: DocumentManager,
     rootUri: DocumentURI?,
@@ -82,7 +88,8 @@ public final class Workspace {
     options: SourceKitLSPServer.Options,
     underlyingBuildSystem: BuildSystem?,
     index uncheckedIndex: UncheckedIndex?,
-    indexDelegate: SourceKitIndexDelegate?
+    indexDelegate: SourceKitIndexDelegate?,
+    indexTaskScheduler: TaskScheduler<UpdateIndexStoreTaskDescription>
   ) async {
     self.documentManager = documentManager
     self.buildSetup = options.buildSetup
@@ -95,6 +102,16 @@ public final class Workspace {
       mainFilesProvider: uncheckedIndex,
       toolchainRegistry: toolchainRegistry
     )
+    if let uncheckedIndex, options.indexOptions.enableBackgroundIndexing {
+      self.semanticIndexManager = SemanticIndexManager(
+        index: uncheckedIndex,
+        buildSystemManager: buildSystemManager,
+        indexTaskScheduler: indexTaskScheduler,
+        indexTaskDidFinish: options.indexOptions.indexTaskDidFinish
+      )
+    } else {
+      self.semanticIndexManager = nil
+    }
     await indexDelegate?.addMainFileChangedCallback { [weak self] in
       await self?.buildSystemManager.mainFilesChanged()
     }
@@ -106,6 +123,9 @@ public final class Workspace {
     }
     // Trigger an initial population of `syntacticTestIndex`.
     await syntacticTestIndex.listOfTestFilesDidChange(buildSystemManager.testFiles())
+    if let semanticIndexManager, let underlyingBuildSystem {
+      await semanticIndexManager.scheduleBackgroundIndex(files: await underlyingBuildSystem.sourceFiles().map(\.uri))
+    }
   }
 
   /// Creates a workspace for a given root `URL`, inferring the `ExternalWorkspace` if possible.
@@ -122,11 +142,16 @@ public final class Workspace {
     options: SourceKitLSPServer.Options,
     compilationDatabaseSearchPaths: [RelativePath],
     indexOptions: IndexOptions = IndexOptions(),
+    indexTaskScheduler: TaskScheduler<UpdateIndexStoreTaskDescription>,
     reloadPackageStatusCallback: @escaping (ReloadPackageStatus) async -> Void
   ) async throws {
     var buildSystem: BuildSystem? = nil
 
     if let rootUrl = rootUri.fileURL, let rootPath = try? AbsolutePath(validating: rootUrl.path) {
+      var options = options
+      if options.indexOptions.enableBackgroundIndexing, options.buildSetup.path == nil {
+        options.buildSetup.path = rootPath.appending(component: ".index-build")
+      }
       func createSwiftPMBuildSystem(rootUrl: URL) async -> SwiftPMBuildSystem? {
         return await SwiftPMBuildSystem(
           url: rootUrl,
@@ -218,7 +243,8 @@ public final class Workspace {
       options: options,
       underlyingBuildSystem: buildSystem,
       index: UncheckedIndex(index),
-      indexDelegate: indexDelegate
+      indexDelegate: indexDelegate,
+      indexTaskScheduler: indexTaskScheduler
     )
   }
 
@@ -258,15 +284,34 @@ public struct IndexOptions {
   /// explicit calls to pollForUnitChangesAndWait().
   public var listenToUnitEvents: Bool
 
+  /// Whether background indexing should be enabled.
+  public var enableBackgroundIndexing: Bool
+
+  /// The percentage of the machine's cores that should at most be used for background indexing.
+  ///
+  /// Setting this to a value < 1 ensures that background indexing doesn't use all CPU resources.
+  public var maxCoresPercentageToUseForBackgroundIndexing: Double
+
+  /// A callback that is called when an index task finishes.
+  ///
+  /// Intended for testing purposes.
+  public var indexTaskDidFinish: (@Sendable (UpdateIndexStoreTaskDescription) -> Void)?
+
   public init(
     indexStorePath: AbsolutePath? = nil,
     indexDatabasePath: AbsolutePath? = nil,
     indexPrefixMappings: [PathPrefixMapping]? = nil,
-    listenToUnitEvents: Bool = true
+    listenToUnitEvents: Bool = true,
+    enableBackgroundIndexing: Bool = false,
+    maxCoresPercentageToUseForBackgroundIndexing: Double = 1,
+    indexTaskDidFinish: (@Sendable (UpdateIndexStoreTaskDescription) -> Void)? = nil
   ) {
     self.indexStorePath = indexStorePath
     self.indexDatabasePath = indexDatabasePath
     self.indexPrefixMappings = indexPrefixMappings
     self.listenToUnitEvents = listenToUnitEvents
+    self.enableBackgroundIndexing = enableBackgroundIndexing
+    self.maxCoresPercentageToUseForBackgroundIndexing = maxCoresPercentageToUseForBackgroundIndexing
+    self.indexTaskDidFinish = indexTaskDidFinish
   }
 }

Tests/SKCoreTests/TaskSchedulerTests.swift

@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 import SKCore
+import SKTestSupport
 import XCTest
 
 final class TaskSchedulerTests: XCTestCase {
@@ -285,24 +286,6 @@ fileprivate actor TaskExecutionRecorder {
   }
 }
 
-/// Wrapper around `DispatchSemaphore` so that Swift Concurrency doesn't complain about the usage of semaphores in the
-/// tests.
-fileprivate struct UnsafeSemaphore {
-  let semaphore = DispatchSemaphore(value: 0)
-
-  func signal(value: Int = 1) {
-    for _ in 0..<value {
-      semaphore.signal()
-    }
-  }
-
-  func wait(value: Int = 1) {
-    for _ in 0..<value {
-      semaphore.wait()
-    }
-  }
-}
-
 fileprivate func runTaskScheduler(
   highPriorityTasks: Int = 4,
   lowPriorityTasks: Int = 2,
@@ -315,7 +298,7 @@ fileprivate func runTaskScheduler(
   )
   let taskExecutionRecorder = TaskExecutionRecorder()
 
-  let allTasksScheduled = UnsafeSemaphore()
+  let allTasksScheduled = WrappedSemaphore()
 
   // Keep scheduler busy so we can schedule all the remaining tasks that we actually want to test.
   // Using a semaphore here is an anti-pattern that should not be used in production since it can lead to priority
@@ -333,7 +316,7 @@ fileprivate func runTaskScheduler(
   // Use a semaphore to wait for the scheduler to reach these very low-priority tasks.
   // Using utility for the priority ensures that these tasks get executed last and using a semaphore ensures that we
   // don't elevate the task's priority by awaiting it.
-  let reachedEnd = UnsafeSemaphore()
+  let reachedEnd = WrappedSemaphore()
   await scheduler.schedule(
     priority: TaskPriority.low,
     id: nil,

Tests/SourceKitLSPTests/BackgroundIndexingTests.swift

@@ -0,0 +1,155 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2024 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import LSPTestSupport
+import LanguageServerProtocol
+import SKTestSupport
+import SourceKitLSP
+import XCTest
+
+fileprivate let backgroundIndexingOptions = SourceKitLSPServer.Options(
+  indexOptions: IndexOptions(enableBackgroundIndexing: true)
+)
+
+final class BackgroundIndexingTests: XCTestCase {
+  func testBackgroundIndexingOfSingleFile() async throws {
+    let project = try await SwiftPMTestProject(
+      files: [
+        "MyFile.swift": """
+        func 1️⃣foo() {}
+        func 2️⃣bar() {
+          3️⃣foo()
+        }
+        """
+      ],
+      serverOptions: backgroundIndexingOptions
+    )
+
+    let (uri, positions) = try project.openDocument("MyFile.swift")
+    let prepare = try await project.testClient.send(
+      CallHierarchyPrepareRequest(textDocument: TextDocumentIdentifier(uri), position: positions["1️⃣"])
+    )
+    let initialItem = try XCTUnwrap(prepare?.only)
+    let calls = try await project.testClient.send(CallHierarchyIncomingCallsRequest(item: initialItem))
+    XCTAssertEqual(
+      calls,
+      [
+        CallHierarchyIncomingCall(
+          from: CallHierarchyItem(
+            name: "bar()",
+            kind: .function,
+            tags: nil,
+            uri: uri,
+            range: Range(positions["2️⃣"]),
+            selectionRange: Range(positions["2️⃣"]),
+            data: .dictionary([
+              "usr": .string("s:9MyLibrary3baryyF"),
+              "uri": .string(uri.stringValue),
+            ])
+          ),
+          fromRanges: [Range(positions["3️⃣"])]
+        )
+      ]
+    )
+  }
+
+  func testBackgroundIndexingOfMultiFileModule() async throws {
+    let project = try await SwiftPMTestProject(
+      files: [
+        "MyFile.swift": """
+        func 1️⃣foo() {}
+        """,
+        "MyOtherFile.swift": """
+        func 2️⃣bar() {
+          3️⃣foo()
+        }
+        """,
+      ],
+      serverOptions: backgroundIndexingOptions
+    )
+
+    let (uri, positions) = try project.openDocument("MyFile.swift")
+    let prepare = try await project.testClient.send(
+      CallHierarchyPrepareRequest(textDocument: TextDocumentIdentifier(uri), position: positions["1️⃣"])
+    )
+    let initialItem = try XCTUnwrap(prepare?.only)
+    let calls = try await project.testClient.send(CallHierarchyIncomingCallsRequest(item: initialItem))
+    XCTAssertEqual(
+      calls,
+      [
+        CallHierarchyIncomingCall(
+          from: CallHierarchyItem(
+            name: "bar()",
+            kind: .function,
+            tags: nil,
+            uri: try project.uri(for: "MyOtherFile.swift"),
+            range: Range(try project.position(of: "2️⃣", in: "MyOtherFile.swift")),
+            selectionRange: Range(try project.position(of: "2️⃣", in: "MyOtherFile.swift")),
+            data: .dictionary([
+              "usr": .string("s:9MyLibrary3baryyF"),
+              "uri": .string(try project.uri(for: "MyOtherFile.swift").stringValue),
+            ])
+          ),
+          fromRanges: [Range(try project.position(of: "3️⃣", in: "MyOtherFile.swift"))]
+        )
+      ]
+    )
+  }
+
+  func testBackgroundIndexingHappensWithLowPriority() async throws {
+    var serverOptions = backgroundIndexingOptions
+    serverOptions.indexOptions.indexTaskDidFinish = { taskDescription in
+      XCTAssert(
+        Task.currentPriority == .low,
+        "\(taskDescription.description) ran with priority \(Task.currentPriority)"
+      )
+    }
+    let project = try await SwiftPMTestProject(
+      files: [
+        "LibA/MyFile.swift": """
+        public func 1️⃣foo() {}
+        """,
+        "LibB/MyOtherFile.swift": """
+        import LibA
+        func 2️⃣bar() {
+          3️⃣foo()
+        }
+        """,
+      ],
+      manifest: """
+        // swift-tools-version: 5.7
+
+        import PackageDescription
+
+        let package = Package(
+          name: "MyLibrary",
+          targets: [
+           .target(name: "LibA"),
+           .target(name: "LibB", dependencies: ["LibA"]),
+          ]
+        )
+        """,
+      serverOptions: serverOptions,
+      pollIndex: false
+    )
+
+    // Wait for indexing to finish without elevating the priority
+    let semaphore = WrappedSemaphore()
+    Task(priority: .low) {
+      await assertNoThrow {
+        try await project.testClient.send(PollIndexRequest())
+      }
+      semaphore.signal()
+    }
+    semaphore.wait()
+  }
+}

Tests/SourceKitLSPTests/BuildSystemTests.swift

@@ -108,7 +108,8 @@ final class BuildSystemTests: XCTestCase {
       options: SourceKitLSPServer.Options.testDefault,
       underlyingBuildSystem: buildSystem,
       index: nil,
-      indexDelegate: nil
+      indexDelegate: nil,
+      indexTaskScheduler: .forTesting
     )
 
     await server.setWorkspaces([(workspace: workspace, isImplicit: false)])