mirror of
https://github.com/apple/swift.git
synced 2025-12-21 12:14:44 +01:00
This patch allows Parser to generate a refined token stream to satisfy tooling's need. For syntax coloring, token stream from lexer is insufficient because (1) we have contextual keywords like get and set; (2) we may allow keywords to be used as argument labels and names; and (3) we need to split tokens like "==<". In this patch, these refinements are directly fulfilled through parsing without additional heuristics. The refined token vector is optionally saved in SourceFile instance.
154 lines
4.3 KiB
C++
154 lines
4.3 KiB
C++
//===--- RawTokenSyntax.cpp - Swift Token Syntax Implementation -----------===//
|
|
//
|
|
// This source file is part of the Swift.org open source project
|
|
//
|
|
// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
|
|
// Licensed under Apache License v2.0 with Runtime Library Exception
|
|
//
|
|
// See https://swift.org/LICENSE.txt for license information
|
|
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
|
#include "swift/Syntax/TokenSyntax.h"
|
|
|
|
using namespace swift;
|
|
using namespace swift::syntax;
|
|
|
|
RawTokenSyntax::RawTokenSyntax()
|
|
: RawSyntax(SyntaxKind::Token, {}, SourcePresence::Missing),
|
|
TokenKind(tok::NUM_TOKENS), Text(OwnedString()),
|
|
LeadingTrivia({}), TrailingTrivia({}) {}
|
|
|
|
RawTokenSyntax::RawTokenSyntax(tok TokenKind, OwnedString Text,
|
|
const SourcePresence Presence)
|
|
: RawSyntax(SyntaxKind::Token, {}, Presence),
|
|
TokenKind(TokenKind),
|
|
Text(Text),
|
|
LeadingTrivia({}), TrailingTrivia({}) {}
|
|
|
|
RawTokenSyntax::RawTokenSyntax(tok TokenKind, OwnedString Text,
|
|
const SourcePresence Presence,
|
|
const Trivia &LeadingTrivia,
|
|
const Trivia &TrailingTrivia)
|
|
: RawSyntax(SyntaxKind::Token, {}, Presence),
|
|
TokenKind(TokenKind), Text(Text),
|
|
LeadingTrivia(LeadingTrivia), TrailingTrivia(TrailingTrivia) {}
|
|
|
|
AbsolutePosition
|
|
RawTokenSyntax::accumulateAbsolutePosition(AbsolutePosition &Pos) const {
|
|
for (auto Leader : LeadingTrivia) {
|
|
Leader.accumulateAbsolutePosition(Pos);
|
|
}
|
|
|
|
auto Start = Pos;
|
|
|
|
Pos.addText(getText());
|
|
|
|
for (auto Trailer : TrailingTrivia) {
|
|
Trailer.accumulateAbsolutePosition(Pos);
|
|
}
|
|
|
|
return Start;
|
|
}
|
|
|
|
void RawTokenSyntax::dumpKind(llvm::raw_ostream &OS) const {
|
|
switch (getTokenKind()) {
|
|
case tok::unknown:
|
|
OS << "unknown";
|
|
break;
|
|
case tok::eof:
|
|
OS << "eof";
|
|
break;
|
|
case tok::code_complete:
|
|
OS << "code_complete";
|
|
break;
|
|
case tok::identifier:
|
|
OS << "identifier";
|
|
break;
|
|
case tok::oper_binary_unspaced:
|
|
OS << "oper_binary_unspaced";
|
|
break;
|
|
case tok::oper_binary_spaced:
|
|
OS << "oper_binary_spaced";
|
|
break;
|
|
case tok::string_interpolation_anchor:
|
|
OS << "string_interpolation_anchor";
|
|
break;
|
|
case tok::contextual_keyword:
|
|
OS << "contextual_keyword";
|
|
break;
|
|
case tok::oper_postfix:
|
|
OS << "oper_postfix";
|
|
break;
|
|
case tok::oper_prefix:
|
|
OS << "oper_prefix";
|
|
break;
|
|
case tok::dollarident:
|
|
OS << "dollarident";
|
|
break;
|
|
case tok::integer_literal:
|
|
OS << "integer_literal";
|
|
break;
|
|
case tok::floating_literal:
|
|
OS << "floating_literal";
|
|
break;
|
|
case tok::string_literal:
|
|
OS << "string_literal";
|
|
break;
|
|
case tok::sil_local_name:
|
|
OS << "sil_local_name";
|
|
break;
|
|
case tok::comment:
|
|
OS << "comment";
|
|
break;
|
|
case tok::NUM_TOKENS:
|
|
OS << "NUM_TOKENS (unset)";
|
|
break;
|
|
#define KEYWORD(X) \
|
|
case tok::kw_##X: \
|
|
OS << "kw_" << #X; \
|
|
break;
|
|
#define PUNCTUATOR(X, Y) \
|
|
case tok::X: \
|
|
OS << #X; \
|
|
break;
|
|
#define POUND_KEYWORD(X) \
|
|
case tok::pound_##X: \
|
|
OS << "pound_" << #X; \
|
|
break;
|
|
#include "swift/Syntax/TokenKinds.def"
|
|
}
|
|
}
|
|
|
|
void RawTokenSyntax::dump(llvm::raw_ostream &OS, unsigned Indent) const {
|
|
auto indent = [&](unsigned Amount) {
|
|
for (decltype(Amount) i = 0; i < Amount; ++i) {
|
|
OS << ' ';
|
|
}
|
|
};
|
|
|
|
indent(Indent);
|
|
|
|
OS << "(token ";
|
|
dumpKind(OS);
|
|
if (isMissing())
|
|
OS << " [missing] ";
|
|
|
|
for (auto Leader : LeadingTrivia) {
|
|
OS << "\n";
|
|
Leader.dump(OS, Indent + 1);
|
|
}
|
|
|
|
OS << "\n";
|
|
indent(Indent + 1);
|
|
OS << "(text=\"" << getText() << "\")";
|
|
|
|
for (auto Trailer : TrailingTrivia) {
|
|
OS << "\n";
|
|
Trailer.dump(OS, Indent + 1);
|
|
}
|
|
OS << ')';
|
|
}
|