Add a debug subcommand that shows the requests that are currently being handled by SourceKit-LSP

Useful to debug where SourceKit-LSP is currently spending its time if semantic functionality seems to be hanging.

Sources/Diagnose/ActiveRequestsCommand.swift

@@ -0,0 +1,100 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2024 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import ArgumentParser
+import Foundation
+import RegexBuilder
+import class TSCBasic.Process
+
+package struct ActiveRequestsCommand: AsyncParsableCommand {
+  package static let configuration: CommandConfiguration = CommandConfiguration(
+    commandName: "active-requests",
+    abstract: "Shows the requests that are currently being handled by sourcekit-lsp.",
+    discussion: "This command only works on macOS."
+  )
+
+  @Option(
+    name: .customLong("log-file"),
+    help: """
+      Instead of reading the currently executing requests from recent system messages, read them from a log file, \
+      generated by `log show`.
+      """
+  )
+  var logFile: String?
+
+  package init() {}
+
+  /// Read the last 3 minutes of OSLog output, including signpost messages.
+  package func readOSLog() async throws -> String {
+    var data = Data()
+    let process = Process(
+      arguments: [
+        "/usr/bin/log",
+        "show",
+        "--last", "3m",
+        "--predicate", #"subsystem = "org.swift.sourcekit-lsp.message-handling" AND process = "sourcekit-lsp""#,
+        "--signpost",
+      ],
+      outputRedirection: .stream(
+        stdout: { data += $0 },
+        stderr: { _ in }
+      )
+    )
+    try process.launch()
+    try await process.waitUntilExit()
+    guard let result = String(data: data, encoding: .utf8) else {
+      throw GenericError("Failed to decode string from OS Log")
+    }
+    return result
+  }
+
+  package func run() async throws {
+    let log: String
+    if let logFile {
+      log = try String(contentsOf: URL(fileURLWithPath: logFile), encoding: .utf8)
+    } else {
+      log = try await readOSLog()
+    }
+    let logParseRegex = Regex {
+      /.*/
+      "[spid 0x"
+      Capture {  // Signpost ID
+        OneOrMore(.hexDigit)
+      }
+      ", process, "
+      ZeroOrMore(.whitespace)
+      Capture {  // Event ("begin", "event", "end")
+        /[a-z]+/
+      }
+      "]"
+      ZeroOrMore(.any)
+    }
+    var messagesBySignpostID: [Substring: [Substring]] = [:]
+    var endedSignposts: Set<Substring> = []
+    for line in log.split(separator: "\n") {
+      guard let match = try logParseRegex.wholeMatch(in: line) else {
+        continue
+      }
+      let (signpostID, event) = (match.1, match.2)
+      messagesBySignpostID[signpostID, default: []].append(line)
+      if event == "end" {
+        endedSignposts.insert(signpostID)
+      }
+    }
+    let activeSignpostMessages =
+      messagesBySignpostID
+      .filter({ !endedSignposts.contains($0.key) })
+      .sorted(by: { $0.key < $1.key })
+      .flatMap(\.value)
+    print(activeSignpostMessages.joined(separator: "\n"))
+  }
+}

Sources/Diagnose/CMakeLists.txt

@@ -1,7 +1,9 @@
 add_library(Diagnose STATIC
+  ActiveRequestsCommand.swift
   CommandLineArgumentsReducer.swift
   DebugCommand.swift
   DiagnoseCommand.swift
+  GenericError.swift
   IndexCommand.swift
   MergeSwiftFiles.swift
   OSLogScraper.swift
@@ -9,7 +11,6 @@ add_library(Diagnose STATIC
   ReduceFrontendCommand.swift
   ReduceSourceKitDRequest.swift
   ReduceSwiftFrontend.swift
-  ReductionError.swift
   ReproducerBundle.swift
   RequestInfo.swift
   RunSourcekitdRequestCommand.swift

Sources/Diagnose/DebugCommand.swift

@@ -17,6 +17,7 @@ package struct DebugCommand: ParsableCommand {
     commandName: "debug",
     abstract: "Commands to debug sourcekit-lsp. Intended for developers of sourcekit-lsp",
     subcommands: [
+      ActiveRequestsCommand.self,
       IndexCommand.self,
       ReduceCommand.self,
       ReduceFrontendCommand.self,

Sources/Diagnose/DiagnoseCommand.swift

@@ -101,7 +101,7 @@ package struct DiagnoseCommand: AsyncParsableCommand {
     #if canImport(OSLog)
     return try OSLogScraper(searchDuration: TimeInterval(osLogScrapeDuration * 60)).getCrashedRequests()
     #else
-    throw ReductionError("Reduction of sourcekitd crashes is not supported on platforms other than macOS")
+    throw GenericError("Reduction of sourcekitd crashes is not supported on platforms other than macOS")
     #endif
   }
 
@@ -214,7 +214,7 @@ package struct DiagnoseCommand: AsyncParsableCommand {
     reportProgress(.collectingLogMessages(progress: 0), message: "Collecting log messages")
     let outputFileUrl = bundlePath.appendingPathComponent("log.txt")
     guard FileManager.default.createFile(atPath: outputFileUrl.path, contents: nil) else {
-      throw ReductionError("Failed to create log.txt")
+      throw GenericError("Failed to create log.txt")
     }
     let fileHandle = try FileHandle(forWritingTo: outputFileUrl)
     var bytesCollected = 0
@@ -307,7 +307,7 @@ package struct DiagnoseCommand: AsyncParsableCommand {
   private func addSwiftVersion(toBundle bundlePath: URL) async throws {
     let outputFileUrl = bundlePath.appendingPathComponent("swift-versions.txt")
     guard FileManager.default.createFile(atPath: outputFileUrl.path, contents: nil) else {
-      throw ReductionError("Failed to create file at \(outputFileUrl)")
+      throw GenericError("Failed to create file at \(outputFileUrl)")
     }
     let fileHandle = try FileHandle(forWritingTo: outputFileUrl)
 
@@ -434,13 +434,13 @@ package struct DiagnoseCommand: AsyncParsableCommand {
     progressUpdate: (_ progress: Double, _ message: String) -> Void
   ) async throws {
     guard let toolchain else {
-      throw ReductionError("Unable to find a toolchain")
+      throw GenericError("Unable to find a toolchain")
     }
     guard let sourcekitd = toolchain.sourcekitd else {
-      throw ReductionError("Unable to find sourcekitd.framework")
+      throw GenericError("Unable to find sourcekitd.framework")
     }
     guard let swiftFrontend = toolchain.swiftFrontend else {
-      throw ReductionError("Unable to find swift-frontend")
+      throw GenericError("Unable to find swift-frontend")
     }
 
     let requestInfo = requestInfo

Sources/Diagnose/GenericError.swift

@@ -10,8 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-/// Generic error that can be thrown if reducing the crash failed in a non-recoverable way.
-struct ReductionError: Error, CustomStringConvertible {
+struct GenericError: Error, CustomStringConvertible {
   let description: String
 
   init(_ description: String) {

Sources/Diagnose/ReduceCommand.swift

@@ -70,10 +70,10 @@ package struct ReduceCommand: AsyncParsableCommand {
   @MainActor
   package func run() async throws {
     guard let sourcekitd = try await toolchain?.sourcekitd else {
-      throw ReductionError("Unable to find sourcekitd.framework")
+      throw GenericError("Unable to find sourcekitd.framework")
     }
     guard let swiftFrontend = try await toolchain?.swiftFrontend else {
-      throw ReductionError("Unable to find sourcekitd.framework")
+      throw GenericError("Unable to find sourcekitd.framework")
     }
 
     let progressBar = PercentProgressAnimation(stream: stderrStreamConcurrencySafe, header: "Reducing sourcekitd issue")

Sources/Diagnose/ReduceFrontendCommand.swift

@@ -78,10 +78,10 @@ package struct ReduceFrontendCommand: AsyncParsableCommand {
   @MainActor
   package func run() async throws {
     guard let sourcekitd = try await toolchain?.sourcekitd else {
-      throw ReductionError("Unable to find sourcekitd.framework")
+      throw GenericError("Unable to find sourcekitd.framework")
     }
     guard let swiftFrontend = try await toolchain?.swiftFrontend else {
-      throw ReductionError("Unable to find swift-frontend")
+      throw GenericError("Unable to find swift-frontend")
     }
 
     let progressBar = PercentProgressAnimation(

Sources/Diagnose/ReduceSwiftFrontend.swift

@@ -19,13 +19,13 @@ package func reduceFrontendIssue(
   let requestInfo = try RequestInfo(frontendArgs: frontendArgs)
   let initialResult = try await executor.run(request: requestInfo)
   guard case .reproducesIssue = initialResult else {
-    throw ReductionError("Unable to reproduce the swift-frontend issue")
+    throw GenericError("Unable to reproduce the swift-frontend issue")
   }
   let mergedSwiftFilesRequestInfo = try await requestInfo.mergeSwiftFiles(using: executor) { progress, message in
     progressUpdate(0, message)
   }
   guard let mergedSwiftFilesRequestInfo else {
-    throw ReductionError("Merging all .swift files did not reproduce the issue. Unable to reduce it.")
+    throw GenericError("Merging all .swift files did not reproduce the issue. Unable to reduce it.")
   }
   return try await mergedSwiftFilesRequestInfo.reduce(using: executor, progressUpdate: progressUpdate)
 }

Sources/Diagnose/RequestInfo.swift

@@ -35,7 +35,7 @@ package struct RequestInfo: Sendable {
     let encoder = JSONEncoder()
     encoder.outputFormatting = .prettyPrinted
     guard var compilerArgs = String(data: try encoder.encode(compilerArgs), encoding: .utf8) else {
-      throw ReductionError("Failed to encode compiler arguments")
+      throw GenericError("Failed to encode compiler arguments")
     }
     // Drop the opening `[` and `]`. The request template already contains them
     compilerArgs = String(compilerArgs.dropFirst().dropLast())
@@ -92,7 +92,7 @@ package struct RequestInfo: Sendable {
       "\""
     }
     guard let sourceFileMatch = requestTemplate.matches(of: sourceFileRegex).only else {
-      throw ReductionError("Failed to find key.sourcefile in the request")
+      throw GenericError("Failed to find key.sourcefile in the request")
     }
     let sourceFilePath = String(sourceFileMatch.1)
     requestTemplate.replace(sourceFileMatch.1, with: "$FILE")
@@ -124,7 +124,7 @@ package struct RequestInfo: Sendable {
         _ = iterator.next()
       case "-filelist":
         guard let fileList = iterator.next() else {
-          throw ReductionError("Expected file path after -filelist command line argument")
+          throw GenericError("Expected file path after -filelist command line argument")
         }
         frontendArgsWithFilelistInlined += try String(contentsOfFile: fileList, encoding: .utf8)
           .split(separator: "\n")

Sources/Diagnose/RunSourcekitdRequestCommand.swift

@@ -78,7 +78,7 @@ package struct RunSourceKitdRequestCommand: AsyncParsableCommand {
         var error: UnsafeMutablePointer<CChar>?
         let req = sourcekitd.api.request_create_from_yaml(buffer.baseAddress!, &error)!
         if let error {
-          throw ReductionError("Failed to parse sourcekitd request from JSON: \(String(cString: error))")
+          throw GenericError("Failed to parse sourcekitd request from JSON: \(String(cString: error))")
         }
         return req
       }

Sources/Diagnose/SourceKitD+RunWithYaml.swift

@@ -19,7 +19,7 @@ extension SourceKitD {
       var error: UnsafeMutablePointer<CChar>?
       let req = api.request_create_from_yaml(buffer.baseAddress!, &error)
       if let error {
-        throw ReductionError("Failed to parse sourcekitd request from YAML: \(String(cString: error))")
+        throw GenericError("Failed to parse sourcekitd request from YAML: \(String(cString: error))")
       }
       return req
     }

Sources/Diagnose/SourceReducer.swift

@@ -117,7 +117,7 @@ fileprivate class SourceReducer {
     case .reduced:
       break
     case .didNotReproduce:
-      throw ReductionError("Initial request info did not reproduce the issue")
+      throw GenericError("Initial request info did not reproduce the issue")
     case .noChange:
       preconditionFailure("The reduction step always returns empty edits and not `done` so we shouldn't hit this")
     }
@@ -658,7 +658,7 @@ fileprivate func getSwiftInterface(
       areFallbackArgs: true
     )
   default:
-    throw ReductionError("Failed to get Swift Interface for \(moduleName)")
+    throw GenericError("Failed to get Swift Interface for \(moduleName)")
   }
 
   // Extract the line containing the source text and parse that using JSON decoder.
@@ -677,7 +677,7 @@ fileprivate func getSwiftInterface(
     return line
   }.only
   guard let quotedSourceText else {
-    throw ReductionError("Failed to decode Swift interface response for \(moduleName)")
+    throw GenericError("Failed to decode Swift interface response for \(moduleName)")
   }
   // Filter control characters. JSONDecoder really doensn't like them and they are likely not important if they occur eg. in a comment.
   let sanitizedData = Data(quotedSourceText.utf8.filter { $0 >= 32 })

Tests/SourceKitLSPTests/PullDiagnosticsTests.swift

@@ -70,8 +70,7 @@ final class PullDiagnosticsTests: XCTestCase {
     }
     let diagnostics = fullReport.items
 
-    XCTAssertEqual(diagnostics.count, 1)
-    let diagnostic = try XCTUnwrap(diagnostics.first)
+    let diagnostic = try XCTUnwrap(diagnostics.only)
     XCTAssert(
       diagnostic.range == Range(positions["1️⃣"]) || diagnostic.range == Range(positions["2️⃣"]),
       "Unexpected range: \(diagnostic.range)"