Files
sourcekit-lsp/Tests/SourceKitLSPTests/SemanticTokensTests.swift

1115 lines
38 KiB
Swift
Raw Permalink Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
//===----------------------------------------------------------------------===//
//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2014 - 2021 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See https://swift.org/LICENSE.txt for license information
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
//
//===----------------------------------------------------------------------===//
@_spi(SourceKitLSP) import LanguageServerProtocol
import SKLogging
import SKOptions
import SKTestSupport
import SKUtilities
import SourceKitD
@_spi(Testing) import SourceKitLSP
import SwiftLanguageService
import XCTest
private typealias Token = SyntaxHighlightingToken
final class SemanticTokensTests: SourceKitLSPTestCase {
func testIntArrayCoding() async throws {
let tokens = SyntaxHighlightingTokens(tokens: [
Token(
start: Position(line: 2, utf16index: 3),
utf16length: 5,
kind: .string
),
Token(
start: Position(line: 4, utf16index: 2),
utf16length: 1,
kind: .interface,
modifiers: [.deprecated, .definition]
),
])
let encoded = tokens.lspEncoded
XCTAssertEqual(
encoded,
[
2, // line delta
3, // char delta
5, // length
SemanticTokenTypes.string.tokenType, // kind
0, // modifiers
2, // line delta
2, // char delta
1, // length
SemanticTokenTypes.interface.tokenType, // kind
SemanticTokenModifiers.deprecated.rawValue | SemanticTokenModifiers.definition.rawValue, // modifiers
]
)
let decoded = SyntaxHighlightingTokens(lspEncodedTokens: encoded)
XCTAssertEqual(decoded.tokens, tokens.tokens)
}
func testRangeSplitting() async throws {
let snapshot = DocumentSnapshot(
uri: DocumentURI(for: .swift),
language: .swift,
version: 0,
lineTable: LineTable(
"""
struct X {
let x: Int
let y: String
}
"""
)
)
let empty = Position(line: 0, utf16index: 1)..<Position(line: 0, utf16index: 1)
XCTAssertEqual(empty.splitToSingleLineRanges(in: snapshot), [])
let multiLine = Position(line: 1, utf16index: 6)..<Position(line: 2, utf16index: 7)
XCTAssertEqual(
multiLine.splitToSingleLineRanges(in: snapshot),
[
Position(line: 1, utf16index: 6)..<Position(line: 1, utf16index: 12),
Position(line: 2, utf16index: 0)..<Position(line: 2, utf16index: 7),
]
)
let emptyLines = Position(line: 2, utf16index: 14)..<Position(line: 5, utf16index: 1)
XCTAssertEqual(
emptyLines.splitToSingleLineRanges(in: snapshot),
[
Position(line: 2, utf16index: 14)..<Position(line: 2, utf16index: 15),
Position(line: 5, utf16index: 0)..<Position(line: 5, utf16index: 1),
]
)
}
func testEmpty() async throws {
try await assertSemanticTokens(
markedContents: "",
expected: []
)
}
func testRanged() async throws {
try await assertSemanticTokens(
markedContents: """
let x = 1
1⃣let 2⃣test = 3⃣20
4⃣let 5⃣a6⃣bc = 333
let y = 4
""",
range: ("1", "6"),
expected: [
TokenSpec(marker: "1", length: 3, kind: .keyword),
TokenSpec(marker: "2", length: 4, kind: .identifier),
TokenSpec(marker: "3", length: 2, kind: .number),
TokenSpec(marker: "4", length: 3, kind: .keyword),
TokenSpec(marker: "5", length: 3, kind: .identifier),
]
)
}
func testLexicalTokens() async throws {
try await assertSemanticTokens(
markedContents: """
1⃣let 2⃣x = 3⃣3
4⃣var 5⃣y = 6"test"
7⃣/* abc */ 8⃣// 123
""",
expected: [
// let x = 3
TokenSpec(marker: "1", length: 3, kind: .keyword),
TokenSpec(marker: "2", length: 1, kind: .identifier),
TokenSpec(marker: "3", length: 1, kind: .number),
// var y = "test"
TokenSpec(marker: "4", length: 3, kind: .keyword),
TokenSpec(marker: "5", length: 1, kind: .identifier),
TokenSpec(marker: "6", length: 6, kind: .string),
// /* abc */ // 123
TokenSpec(marker: "7", length: 9, kind: .comment),
TokenSpec(marker: "8", length: 6, kind: .comment),
]
)
}
func testLexicalTokensForMultiLineComments() async throws {
try await assertSemanticTokens(
markedContents: """
1⃣let 2⃣x = 3⃣3 4⃣/*
5⃣let x = 12
6⃣*/
""",
expected: [
TokenSpec(marker: "1", length: 3, kind: .keyword),
TokenSpec(marker: "2", length: 1, kind: .identifier),
TokenSpec(marker: "3", length: 1, kind: .number),
// Multi-line comments are split into single-line tokens
TokenSpec(marker: "4", length: 2, kind: .comment),
TokenSpec(marker: "5", length: 10, kind: .comment),
TokenSpec(marker: "6", length: 2, kind: .comment),
]
)
}
func testLexicalTokensForDocComments() async throws {
try await assertSemanticTokens(
markedContents: """
1⃣/** abc */
2⃣/// def
""",
expected: [
TokenSpec(marker: "1", length: 10, kind: .comment, modifiers: .documentation),
TokenSpec(marker: "2", length: 7, kind: .comment, modifiers: .documentation),
]
)
}
func testLexicalTokensForBackticks() async throws {
try await assertSemanticTokens(
markedContents: """
1⃣var 2⃣`if` = 3⃣20
""",
expected: [
TokenSpec(marker: "1", length: 3, kind: .keyword),
TokenSpec(marker: "2", length: 4, kind: .identifier),
TokenSpec(marker: "3", length: 2, kind: .number),
]
)
try await assertSemanticTokens(
markedContents: """
1⃣let 2⃣`else` = 3⃣3
""",
expected: [
TokenSpec(marker: "1", length: 3, kind: .keyword),
TokenSpec(marker: "2", length: 6, kind: .identifier),
TokenSpec(marker: "3", length: 1, kind: .number),
]
)
try await assertSemanticTokens(
markedContents: """
1⃣let `2⃣onLeft = ()
""",
expected: [
TokenSpec(marker: "1", length: 3, kind: .keyword),
TokenSpec(marker: "2", length: 6, kind: .identifier),
]
)
try await assertSemanticTokens(
markedContents: """
1⃣let 2⃣onRight` = ()
""",
expected: [
TokenSpec(marker: "1", length: 3, kind: .keyword),
TokenSpec(marker: "2", length: 7, kind: .identifier),
]
)
}
func testSemanticTokens() async throws {
try await assertSemanticTokens(
markedContents: """
1⃣struct 2⃣X {}
3⃣let 4⃣x = 5⃣X()
6⃣let 7⃣y = 8⃣x 9⃣+ 🔟x
""",
expected: [
// struct X {}
TokenSpec(marker: "1", length: 6, kind: .keyword),
TokenSpec(marker: "2", length: 1, kind: .identifier),
// let x = X()
TokenSpec(marker: "3", length: 3, kind: .keyword),
TokenSpec(marker: "4", length: 1, kind: .identifier),
TokenSpec(marker: "5", length: 1, kind: .struct),
// let y = x + x
TokenSpec(marker: "6", length: 3, kind: .keyword),
TokenSpec(marker: "7", length: 1, kind: .identifier),
TokenSpec(marker: "8", length: 1, kind: .variable),
TokenSpec(marker: "9", length: 1, kind: .operator),
TokenSpec(marker: "🔟", length: 1, kind: .variable),
]
)
try await assertSemanticTokens(
markedContents: """
1⃣func 2⃣a() {}
3⃣let 4⃣b = {}
5⃣a()
6⃣b()
""",
expected: [
// func a() {}
TokenSpec(marker: "1", length: 4, kind: .keyword),
TokenSpec(marker: "2", length: 1, kind: .identifier),
// let b = {}
TokenSpec(marker: "3", length: 3, kind: .keyword),
TokenSpec(marker: "4", length: 1, kind: .identifier),
// a()
TokenSpec(marker: "5", length: 1, kind: .function),
// b()
TokenSpec(marker: "6", length: 1, kind: .variable),
]
)
}
func testSemanticTokensForProtocols() async throws {
try await assertSemanticTokens(
markedContents: """
1⃣protocol 2⃣X {}
3⃣class 4⃣Y: 5⃣X {}
6⃣let 7⃣y: 8⃣Y = 9⃣X()
""",
expected: [
// protocol X {}
TokenSpec(marker: "1", length: 8, kind: .keyword),
TokenSpec(marker: "2", length: 1, kind: .identifier),
// class Y: X {}
TokenSpec(marker: "3", length: 5, kind: .keyword),
TokenSpec(marker: "4", length: 1, kind: .identifier),
TokenSpec(marker: "5", length: 1, kind: .interface),
// let y: Y = X()
TokenSpec(marker: "6", length: 3, kind: .keyword),
TokenSpec(marker: "7", length: 1, kind: .identifier),
TokenSpec(marker: "8", length: 1, kind: .class),
TokenSpec(marker: "9", length: 1, kind: .interface),
]
)
try await assertSemanticTokens(
markedContents: """
1⃣protocol 2⃣X {}
3⃣func 4⃣f<5⃣T: 6⃣X>() {}
""",
expected: [
// protocol X {}
TokenSpec(marker: "1", length: 8, kind: .keyword),
TokenSpec(marker: "2", length: 1, kind: .identifier),
// func f<T: X>() {}
TokenSpec(marker: "3", length: 4, kind: .keyword),
TokenSpec(marker: "4", length: 1, kind: .identifier),
TokenSpec(marker: "5", length: 1, kind: .identifier),
TokenSpec(marker: "6", length: 1, kind: .interface),
]
)
}
func testSemanticTokensForFunctionSignatures() async throws {
try await assertSemanticTokens(
markedContents: "1⃣func 2⃣f(3⃣x: 4⃣Int, _ 5⃣y: 6⃣String) {}",
expected: [
TokenSpec(marker: "1", length: 4, kind: .keyword),
TokenSpec(marker: "2", length: 1, kind: .identifier),
TokenSpec(marker: "3", length: 1, kind: .function, modifiers: .parameterLabel),
TokenSpec(marker: "4", length: 3, kind: .struct, modifiers: .defaultLibrary),
TokenSpec(marker: "5", length: 1, kind: .identifier),
TokenSpec(marker: "6", length: 6, kind: .struct, modifiers: .defaultLibrary),
]
)
}
func testSemanticTokensForFunctionSignaturesWithEmoji() async throws {
try await assertSemanticTokens(
markedContents: "1⃣func 2⃣x👍y() {}",
expected: [
TokenSpec(marker: "1", length: 4, kind: .keyword),
TokenSpec(marker: "2", length: 4, kind: .identifier),
]
)
}
func testSemanticTokensForStaticMethods() async throws {
try await assertSemanticTokens(
markedContents: """
1⃣class 2⃣X {
3⃣static 4⃣func 5⃣f() {}
}
6⃣X.7⃣f()
""",
expected: [
// class X
TokenSpec(marker: "1", length: 5, kind: .keyword),
TokenSpec(marker: "2", length: 1, kind: .identifier),
// static func f() {}
TokenSpec(marker: "3", length: 6, kind: .keyword),
TokenSpec(marker: "4", length: 4, kind: .keyword),
TokenSpec(marker: "5", length: 1, kind: .identifier),
// X.f()
TokenSpec(marker: "6", length: 1, kind: .class),
TokenSpec(marker: "7", length: 1, kind: .method, modifiers: .static),
]
)
try await assertSemanticTokens(
markedContents: """
1⃣class 2⃣X {
3⃣class 4⃣func 5⃣g() {}
}
6⃣X.7⃣g()
""",
expected: [
// class X
TokenSpec(marker: "1", length: 5, kind: .keyword),
TokenSpec(marker: "2", length: 1, kind: .identifier),
// class func g() {}
TokenSpec(marker: "3", length: 5, kind: .keyword),
TokenSpec(marker: "4", length: 4, kind: .keyword),
TokenSpec(marker: "5", length: 1, kind: .identifier),
// X.f()
TokenSpec(marker: "6", length: 1, kind: .class),
TokenSpec(marker: "7", length: 1, kind: .method, modifiers: .static),
]
)
}
func testSemanticTokensForEnumMembers() async throws {
try await assertSemanticTokens(
markedContents: """
1⃣enum 2⃣Maybe<3⃣T> {
4⃣case 5⃣none
}
6⃣let 7⃣x = 8⃣Maybe<9⃣String>.🔟none
""",
expected: [
// enum Maybe<T>
TokenSpec(marker: "1", length: 4, kind: .keyword),
TokenSpec(marker: "2", length: 5, kind: .identifier),
TokenSpec(marker: "3", length: 1, kind: .identifier),
// case none
TokenSpec(marker: "4", length: 4, kind: .keyword),
TokenSpec(marker: "5", length: 4, kind: .identifier),
// let x = Maybe<String>.none
TokenSpec(marker: "6", length: 3, kind: .keyword),
TokenSpec(marker: "7", length: 1, kind: .identifier),
TokenSpec(marker: "8", length: 5, kind: .enum),
TokenSpec(marker: "9", length: 6, kind: .struct, modifiers: .defaultLibrary),
TokenSpec(marker: "🔟", length: 4, kind: .enumMember),
]
)
try await assertSemanticTokens(
markedContents: """
1⃣enum 2⃣Maybe<3⃣T> {
4⃣case 5⃣some(6⃣T)
}
7⃣let 8⃣y: 9⃣Maybe = .🔟some(0⃣42)
""",
expected: [
// enum Maybe<T>
TokenSpec(marker: "1", length: 4, kind: .keyword),
TokenSpec(marker: "2", length: 5, kind: .identifier),
TokenSpec(marker: "3", length: 1, kind: .identifier),
// case some
TokenSpec(marker: "4", length: 4, kind: .keyword),
TokenSpec(marker: "5", length: 4, kind: .identifier),
TokenSpec(marker: "6", length: 1, kind: .typeParameter),
// let y: Maybe = .some(42)
TokenSpec(marker: "7", length: 3, kind: .keyword),
TokenSpec(marker: "8", length: 1, kind: .identifier),
TokenSpec(marker: "9", length: 5, kind: .enum),
TokenSpec(marker: "🔟", length: 4, kind: .enumMember),
TokenSpec(marker: "0", length: 2, kind: .number),
]
)
}
func testRegexSemanticTokens() async throws {
try await assertSemanticTokens(
markedContents: """
1⃣let 2⃣r = 3⃣/a[bc]*/
""",
expected: [
TokenSpec(marker: "1", length: 3, kind: .keyword),
TokenSpec(marker: "2", length: 1, kind: .identifier),
TokenSpec(marker: "3", length: 8, kind: .regexp),
]
)
}
func testOperatorDeclaration() async throws {
try await assertSemanticTokens(
markedContents: """
1⃣infix 2⃣operator 3⃣?= :4⃣ComparisonPrecedence
""",
expected: [
TokenSpec(marker: "1", length: 5, kind: .keyword),
TokenSpec(marker: "2", length: 8, kind: .keyword),
TokenSpec(marker: "3", length: 2, kind: .operator),
TokenSpec(marker: "4", length: 20, kind: .identifier),
]
)
}
func testEmptyEdit() async throws {
let testClient = try await TestSourceKitLSPClient()
let uri = DocumentURI(for: .swift)
let positions = testClient.openDocument(
"""
1⃣l0⃣et 2⃣x: 3⃣String = 4"test"
5⃣var 6⃣y = 7⃣123
""",
uri: uri
)
let expectedTokens = [
TokenSpec(marker: "1", length: 3, kind: .keyword),
TokenSpec(marker: "2", length: 1, kind: .identifier),
TokenSpec(marker: "3", length: 6, kind: .struct, modifiers: .defaultLibrary),
TokenSpec(marker: "4", length: 6, kind: .string),
TokenSpec(marker: "5", length: 3, kind: .keyword),
TokenSpec(marker: "6", length: 1, kind: .identifier),
TokenSpec(marker: "7", length: 3, kind: .number),
]
try await assertSemanticTokens(uri: uri, in: testClient, positions: positions, expected: expectedTokens)
testClient.send(
DidChangeTextDocumentNotification(
textDocument: VersionedTextDocumentIdentifier(uri, version: 2),
contentChanges: [TextDocumentContentChangeEvent(range: Range(positions["0"]), text: "")]
)
)
try await assertSemanticTokens(uri: uri, in: testClient, positions: positions, expected: expectedTokens)
}
func testReplaceUntilMiddleOfToken() async throws {
let testClient = try await TestSourceKitLSPClient()
let uri = DocumentURI(for: .swift)
let positions = testClient.openDocument(
"""
1⃣var 2⃣test = 3⃣454⃣67
""",
uri: uri
)
try await assertSemanticTokens(
uri: uri,
in: testClient,
positions: positions,
expected: [
TokenSpec(marker: "1", length: 3, kind: .keyword),
TokenSpec(marker: "2", length: 4, kind: .identifier),
TokenSpec(marker: "3", length: 4, kind: .number),
]
)
testClient.send(
DidChangeTextDocumentNotification(
textDocument: VersionedTextDocumentIdentifier(uri, version: 2),
contentChanges: [TextDocumentContentChangeEvent(range: positions["3"]..<positions["4"], text: " 1")]
)
)
let positionsAfterEdits = DocumentPositions(
markedText: """
1⃣var 2⃣test = 3⃣167
"""
)
try await assertSemanticTokens(
uri: uri,
in: testClient,
positions: positionsAfterEdits,
expected: [
TokenSpec(marker: "1", length: 3, kind: .keyword),
TokenSpec(marker: "2", length: 4, kind: .identifier),
TokenSpec(marker: "3", length: 3, kind: .number),
]
)
}
func testReplaceUntilEndOfToken() async throws {
let testClient = try await TestSourceKitLSPClient()
let uri = DocumentURI(for: .swift)
let positions = testClient.openDocument(
"""
1⃣fatalError2⃣(3"xyz"4⃣)
""",
uri: uri
)
try await assertSemanticTokens(
uri: uri,
in: testClient,
positions: positions,
expected: [
TokenSpec(marker: "1", length: 10, kind: .function, modifiers: .defaultLibrary),
TokenSpec(marker: "3", length: 5, kind: .string),
]
)
testClient.send(
DidChangeTextDocumentNotification(
textDocument: VersionedTextDocumentIdentifier(uri, version: 2),
contentChanges: [
TextDocumentContentChangeEvent(
range: positions["2"]..<positions["4"],
text: """
("test"
"""
)
]
)
)
let positionsAfterEdits = DocumentPositions(
markedText: """
1⃣fatalError2⃣(3"test"4⃣)
"""
)
try await assertSemanticTokens(
uri: uri,
in: testClient,
positions: positionsAfterEdits,
expected: [
TokenSpec(marker: "1", length: 10, kind: .function, modifiers: .defaultLibrary),
TokenSpec(marker: "3", length: 6, kind: .string),
]
)
}
func testInsertSpaceBeforeToken() async throws {
let testClient = try await TestSourceKitLSPClient()
let uri = DocumentURI(for: .swift)
let positions = testClient.openDocument(
"""
1⃣let 2⃣x: 3⃣String = 4"test"
""",
uri: uri
)
let expectedTokens = [
TokenSpec(marker: "1", length: 3, kind: .keyword),
TokenSpec(marker: "2", length: 1, kind: .identifier),
TokenSpec(marker: "3", length: 6, kind: .struct, modifiers: .defaultLibrary),
TokenSpec(marker: "4", length: 6, kind: .string),
]
try await assertSemanticTokens(uri: uri, in: testClient, positions: positions, expected: expectedTokens)
testClient.send(
DidChangeTextDocumentNotification(
textDocument: VersionedTextDocumentIdentifier(uri, version: 2),
contentChanges: [TextDocumentContentChangeEvent(range: Range(positions["1"]), text: " ")]
)
)
let positionsAfterEdits = DocumentPositions(
markedText: """
1⃣let 2⃣x: 3⃣String = 4"test"
"""
)
try await assertSemanticTokens(uri: uri, in: testClient, positions: positionsAfterEdits, expected: expectedTokens)
}
func testInsertSpaceAfterToken() async throws {
let testClient = try await TestSourceKitLSPClient()
let uri = DocumentURI(for: .swift)
let positions = testClient.openDocument(
"""
1⃣var 2⃣x = 3⃣04
""",
uri: uri
)
let expectedTokens = [
TokenSpec(marker: "1", length: 3, kind: .keyword),
TokenSpec(marker: "2", length: 1, kind: .identifier),
TokenSpec(marker: "3", length: 1, kind: .number),
]
try await assertSemanticTokens(uri: uri, in: testClient, positions: positions, expected: expectedTokens)
testClient.send(
DidChangeTextDocumentNotification(
textDocument: VersionedTextDocumentIdentifier(uri, version: 2),
contentChanges: [TextDocumentContentChangeEvent(range: Range(positions["4"]), text: " ")]
)
)
try await assertSemanticTokens(uri: uri, in: testClient, positions: positions, expected: expectedTokens)
}
func testInsertNewline() async throws {
let testClient = try await TestSourceKitLSPClient()
let uri = DocumentURI(for: .swift)
let positions = testClient.openDocument(
"""
1⃣fatalError(2"123")
""",
uri: uri
)
let expectedTokens = [
TokenSpec(marker: "1", length: 10, kind: .function, modifiers: .defaultLibrary),
TokenSpec(marker: "2", length: 5, kind: .string),
]
try await assertSemanticTokens(uri: uri, in: testClient, positions: positions, expected: expectedTokens)
testClient.send(
DidChangeTextDocumentNotification(
textDocument: VersionedTextDocumentIdentifier(uri, version: 2),
contentChanges: [TextDocumentContentChangeEvent(range: Range(positions["1"]), text: "\n")]
)
)
let positionsAfterEdit = DocumentPositions(
markedText: """
1⃣fatalError(2"123")
"""
)
try await assertSemanticTokens(uri: uri, in: testClient, positions: positionsAfterEdit, expected: expectedTokens)
}
func testRemoveNewline() async throws {
let testClient = try await TestSourceKitLSPClient()
let uri = DocumentURI(for: .swift)
let positions = testClient.openDocument(
"""
1⃣let 2⃣x =3
4"abc"
""",
uri: uri
)
let expectedTokens = [
TokenSpec(marker: "1", length: 3, kind: .keyword),
TokenSpec(marker: "2", length: 1, kind: .identifier),
TokenSpec(marker: "4", length: 5, kind: .string),
]
try await assertSemanticTokens(uri: uri, in: testClient, positions: positions, expected: expectedTokens)
testClient.send(
DidChangeTextDocumentNotification(
textDocument: VersionedTextDocumentIdentifier(uri, version: 2),
contentChanges: [TextDocumentContentChangeEvent(range: positions["3"]..<positions["4"], text: " ")]
)
)
let positionsAfterEdit = DocumentPositions(
markedText: """
1⃣let 2⃣x = 4"abc"
"""
)
try await assertSemanticTokens(uri: uri, in: testClient, positions: positionsAfterEdit, expected: expectedTokens)
}
func testInsertTokens() async throws {
let testClient = try await TestSourceKitLSPClient()
let uri = DocumentURI(for: .swift)
let positions = testClient.openDocument(
"""
1⃣let 2⃣x =3
4"abc"
""",
uri: uri
)
try await assertSemanticTokens(
uri: uri,
in: testClient,
positions: positions,
expected: [
TokenSpec(marker: "1", length: 3, kind: .keyword),
TokenSpec(marker: "2", length: 1, kind: .identifier),
TokenSpec(marker: "4", length: 5, kind: .string),
]
)
testClient.send(
DidChangeTextDocumentNotification(
textDocument: VersionedTextDocumentIdentifier(uri, version: 2),
contentChanges: [TextDocumentContentChangeEvent(range: positions["3"]..<positions["4"], text: #" "test" + "#)]
)
)
let positionsAfterEdits = DocumentPositions(
markedText: """
1⃣let 2⃣x = 3"test" 4⃣+ 5"abc"
"""
)
try await assertSemanticTokens(
uri: uri,
in: testClient,
positions: positionsAfterEdits,
expected: [
TokenSpec(marker: "1", length: 3, kind: .keyword),
TokenSpec(marker: "2", length: 1, kind: .identifier),
TokenSpec(marker: "3", length: 6, kind: .string),
TokenSpec(marker: "4", length: 1, kind: .method, modifiers: [.defaultLibrary, .static]),
TokenSpec(marker: "5", length: 5, kind: .string),
]
)
}
func testSemanticMultiEdit() async throws {
let testClient = try await TestSourceKitLSPClient()
let uri = DocumentURI(for: .swift)
let positions = testClient.openDocument(
"""
1⃣let 2⃣x3⃣ = 4"abc"
5⃣let 6⃣y = 7⃣x8
""",
uri: uri
)
try await assertSemanticTokens(
uri: uri,
in: testClient,
positions: positions,
expected: [
TokenSpec(marker: "1", length: 3, kind: .keyword),
TokenSpec(marker: "2", length: 1, kind: .identifier),
TokenSpec(marker: "4", length: 5, kind: .string),
TokenSpec(marker: "5", length: 3, kind: .keyword),
TokenSpec(marker: "6", length: 1, kind: .identifier),
TokenSpec(marker: "7", length: 1, kind: .variable),
]
)
let newName = "renamed"
testClient.send(
DidChangeTextDocumentNotification(
textDocument: VersionedTextDocumentIdentifier(uri, version: 2),
contentChanges: [
TextDocumentContentChangeEvent(range: positions["2"]..<positions["3"], text: newName),
TextDocumentContentChangeEvent(range: positions["7"]..<positions["8"], text: newName),
]
)
)
let positionsAfterEdits = DocumentPositions(
markedText: """
1⃣let 2⃣renamed = 4"abc"
5⃣let 6⃣y = 7⃣renamed
"""
)
try await assertSemanticTokens(
uri: uri,
in: testClient,
positions: positionsAfterEdits,
expected: [
TokenSpec(marker: "1", length: 3, kind: .keyword),
TokenSpec(marker: "2", length: 7, kind: .identifier),
TokenSpec(marker: "4", length: 5, kind: .string),
TokenSpec(marker: "5", length: 3, kind: .keyword),
TokenSpec(marker: "6", length: 1, kind: .identifier),
TokenSpec(marker: "7", length: 7, kind: .variable),
]
)
}
func testActor() async throws {
try await assertSemanticTokens(
markedContents: """
1⃣actor 2⃣MyActor {}
3⃣func 4⃣t(5⃣x: 6⃣MyActor) {}
""",
expected: [
TokenSpec(marker: "1", length: 5, kind: .keyword),
TokenSpec(marker: "2", length: 7, kind: .identifier),
TokenSpec(marker: "3", length: 4, kind: .keyword),
TokenSpec(marker: "4", length: 1, kind: .identifier),
TokenSpec(marker: "5", length: 1, kind: .function, modifiers: .parameterLabel),
TokenSpec(marker: "6", length: 7, kind: .actor),
]
)
}
func testArgumentLabels() async throws {
try await assertSemanticTokens(
markedContents: """
1⃣func 2⃣foo(3⃣arg: 4⃣Int) {}
5⃣foo(6⃣arg: 7⃣1)
""",
expected: [
TokenSpec(marker: "1", length: 4, kind: .keyword),
TokenSpec(marker: "2", length: 3, kind: .identifier),
TokenSpec(marker: "3", length: 3, kind: .function, modifiers: .parameterLabel),
TokenSpec(marker: "4", length: 3, kind: .struct, modifiers: .defaultLibrary),
TokenSpec(marker: "5", length: 3, kind: .function),
TokenSpec(marker: "6", length: 3, kind: .function, modifiers: .parameterLabel),
TokenSpec(marker: "7", length: 1, kind: .number),
]
)
}
func testFunctionDeclarationWithFirstAndSecondName() async throws {
try await assertSemanticTokens(
markedContents: """
1⃣func 2⃣foo(3⃣arg 4⃣internalName: 5⃣Int) {}
""",
expected: [
TokenSpec(marker: "1", length: 4, kind: .keyword),
TokenSpec(marker: "2", length: 3, kind: .identifier),
TokenSpec(marker: "3", length: 3, kind: .function, modifiers: .parameterLabel),
TokenSpec(marker: "4", length: 12, kind: .identifier),
TokenSpec(marker: "5", length: 3, kind: .struct, modifiers: .defaultLibrary),
]
)
}
func testRawIdentifier() async throws {
try await SkipUnless.haveRawIdentifiers()
try await assertSemanticTokens(
markedContents: """
1⃣func 2⃣`square returns x * x`() {}
3⃣`square returns x * x`()
""",
expected: [
TokenSpec(marker: "1", length: 4, kind: .keyword),
TokenSpec(marker: "2", length: 22, kind: .identifier),
TokenSpec(marker: "3", length: 22, kind: .function),
]
)
try await assertSemanticTokens(
markedContents: """
1⃣struct 2⃣S {
3⃣static 4⃣var 5⃣`foo bar` = 6⃣0
}
7⃣S.8️⃣`foo bar`
""",
expected: [
TokenSpec(marker: "1", length: 6, kind: .keyword),
TokenSpec(marker: "2", length: 1, kind: .identifier),
TokenSpec(marker: "3", length: 6, kind: .keyword),
TokenSpec(marker: "4", length: 3, kind: .keyword),
TokenSpec(marker: "5", length: 9, kind: .identifier),
TokenSpec(marker: "6", length: 1, kind: .number),
TokenSpec(marker: "7", length: 1, kind: .struct),
TokenSpec(marker: "8", length: 9, kind: .property),
]
)
}
func testCloseAndReopenDocumentWithSameDocumentVersion() async throws {
// When neovim detects a change of the document on-disk (eg. caused by git operations). It closes the document and
// re-opens it with the same document version but different contents. Check that we don't re-use the syntax tree of
// the previously opened document.
let testClient = try await TestSourceKitLSPClient()
let uri = DocumentURI(for: .swift)
let initialPositions = testClient.openDocument(
"""
1⃣import 2⃣Foo
3⃣func 4⃣bar() {}
""",
uri: uri
)
let initialTokens = try await testClient.send(
DocumentSemanticTokensRequest(textDocument: TextDocumentIdentifier(uri))
)
XCTAssertEqual(
SyntaxHighlightingTokens(lspEncodedTokens: try unwrap(initialTokens).data).tokens,
[
Token(start: initialPositions["1"], utf16length: 6, kind: .keyword),
Token(start: initialPositions["2"], utf16length: 3, kind: .identifier),
Token(start: initialPositions["3"], utf16length: 4, kind: .keyword),
Token(start: initialPositions["4"], utf16length: 3, kind: .identifier),
]
)
testClient.send(DidCloseTextDocumentNotification(textDocument: TextDocumentIdentifier(uri)))
let reopenedPositions = testClient.openDocument(
"""
1⃣func 2⃣bar() {}
""",
uri: uri
)
let reopenedTokens = try await testClient.send(
DocumentSemanticTokensRequest(textDocument: TextDocumentIdentifier(uri))
)
XCTAssertEqual(
SyntaxHighlightingTokens(lspEncodedTokens: try unwrap(reopenedTokens).data).tokens,
[
Token(start: reopenedPositions["1"], utf16length: 4, kind: .keyword),
Token(start: reopenedPositions["2"], utf16length: 3, kind: .identifier),
]
)
}
func testClang() async throws {
try await assertSemanticTokens(
markedContents: """
int 1⃣main() {}
""",
language: .c,
expected: [
TokenSpec(marker: "1", length: 4, kind: .function, modifiers: [.declaration, .definition, .globalScope])
]
)
}
func testImplicitCancellationOnEdit() async throws {
let testClient = try await TestSourceKitLSPClient(
hooks: Hooks(preHandleRequest: { request in
if request is DocumentSemanticTokensRequest {
while !Task.isCancelled {
try? await Task.sleep(for: .seconds(1))
}
}
})
)
let uri = DocumentURI(for: .swift)
let positions = testClient.openDocument("1", uri: uri)
let receivedSemanticTokensResponse = self.expectation(description: "Received semantic tokens response")
testClient.send(DocumentSemanticTokensRequest(textDocument: TextDocumentIdentifier(uri))) { result in
XCTAssertEqual(result, .failure(ResponseError.cancelled))
receivedSemanticTokensResponse.fulfill()
}
testClient.send(
DidChangeTextDocumentNotification(
textDocument: VersionedTextDocumentIdentifier(uri, version: 2),
contentChanges: [TextDocumentContentChangeEvent(range: Range(positions["1"]), text: "let x = 1")]
)
)
try await fulfillmentOfOrThrow(receivedSemanticTokensResponse)
}
func testNoImplicitCancellationOnEditIfImplicitCancellationIsDisabled() async throws {
try SkipUnless.longTestsEnabled()
let testClient = try await TestSourceKitLSPClient(
options: SourceKitLSPOptions(cancelTextDocumentRequestsOnEditAndClose: false),
hooks: Hooks(preHandleRequest: { request in
if request is DocumentSemanticTokensRequest {
// Sleep long enough for the edit to be handled
try? await Task.sleep(for: .seconds(2))
}
})
)
let uri = DocumentURI(for: .swift)
let positions = testClient.openDocument("1", uri: uri)
let receivedSemanticTokensResponse = self.expectation(description: "Received semantic tokens response")
testClient.send(DocumentSemanticTokensRequest(textDocument: TextDocumentIdentifier(uri))) { result in
XCTAssertEqual(result, .success(DocumentSemanticTokensResponse(data: [])))
receivedSemanticTokensResponse.fulfill()
}
testClient.send(
DidChangeTextDocumentNotification(
textDocument: VersionedTextDocumentIdentifier(uri, version: 2),
contentChanges: [TextDocumentContentChangeEvent(range: Range(positions["1"]), text: "let x = 1")]
)
)
try await fulfillmentOfOrThrow(receivedSemanticTokensResponse)
}
}
private struct TokenSpec {
let marker: String
let length: Int
let kind: SemanticTokenTypes
let modifiers: SemanticTokenModifiers
init(
marker: String,
length: Int,
kind: SemanticTokenTypes,
modifiers: SemanticTokenModifiers = []
) {
self.marker = marker
self.length = length
self.kind = kind
self.modifiers = modifiers
}
}
private func assertSemanticTokens(
markedContents: String,
language: Language = .swift,
range: (startMarker: String, endMarker: String)? = nil,
expected: [TokenSpec],
file: StaticString = #filePath,
line: UInt = #line
) async throws {
let testClient = try await TestSourceKitLSPClient()
let uri = DocumentURI(for: language)
let positions = testClient.openDocument(markedContents, uri: uri)
let response: DocumentSemanticTokensResponse?
if let range {
response = try await testClient.send(
DocumentSemanticTokensRangeRequest(
textDocument: TextDocumentIdentifier(uri),
range: positions[range.startMarker]..<positions[range.endMarker]
)
)
} else {
response = try await testClient.send(DocumentSemanticTokensRequest(textDocument: TextDocumentIdentifier(uri)))
}
let expectedTokens = expected.map {
Token(start: positions[$0.marker], utf16length: $0.length, kind: $0.kind, modifiers: $0.modifiers)
}
XCTAssertEqual(
SyntaxHighlightingTokens(lspEncodedTokens: try unwrap(response, file: file, line: line).data).tokens,
expectedTokens,
file: file,
line: line
)
}
private func assertSemanticTokens(
uri: DocumentURI,
in testClient: TestSourceKitLSPClient,
positions: DocumentPositions,
expected: [TokenSpec],
file: StaticString = #filePath,
line: UInt = #line
) async throws {
let response = try await unwrap(
testClient.send(DocumentSemanticTokensRequest(textDocument: TextDocumentIdentifier(uri))),
file: file,
line: line
)
let expectedTokens = expected.map {
Token(start: positions[$0.marker], utf16length: $0.length, kind: $0.kind, modifiers: $0.modifiers)
}
XCTAssertEqual(
SyntaxHighlightingTokens(lspEncodedTokens: response.data).tokens,
expectedTokens,
file: file,
line: line
)
}