averello/swift-mirror
1
0
Fork 0
mirror of https://github.com/apple/swift.git synced 2025-12-21 12:14:44 +01:00
Code Issues Packages Projects Releases Wiki Activity

[libSyntax] Restructure RawSyntax to more closely resemble the SwiftSyntax implementation

Browse Source
This commit is contained in:
Alex Hoppen
2021-01-15 15:35:23 +01:00
parent fb70ab4279
commit 8bb1167e21
27 changed files with 743 additions and 601 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
include/swift/Parse/Parser.h
View File

@@ -59,7 +59,6 @@ namespace swift {
struct EnumElementInfo;
namespace syntax {
class AbsolutePosition;
class RawSyntax;
enum class SyntaxKind;
}// end of syntax namespace
@@ -1817,14 +1816,10 @@ bool isKeywordPossibleDeclStart(const Token &Tok);
/// Lex and return a vector of `TokenSyntax` tokens, which include
/// leading and trailing trivia.
std::vector<std::pair<RC<syntax::RawSyntax>,
syntax::AbsolutePosition>>
tokenizeWithTrivia(const LangOptions &LangOpts,
const SourceManager &SM,
unsigned BufferID,
unsigned Offset = 0,
unsigned EndOffset = 0,
DiagnosticEngine *Diags = nullptr);
std::vector<std::pair<RC<syntax::RawSyntax>, syntax::AbsoluteOffsetPosition>>
tokenizeWithTrivia(const LangOptions &LangOpts, const SourceManager &SM,
unsigned BufferID, unsigned Offset = 0,
unsigned EndOffset = 0, DiagnosticEngine *Diags = nullptr);
} // end namespace swift
#endif

4
include/swift/Parse/SyntaxParsingCache.h
View File

@@ -48,8 +48,8 @@ struct SourceEdit {
};
struct SyntaxReuseRegion {
AbsolutePosition Start;
AbsolutePosition End;
AbsoluteOffsetPosition Start;
AbsoluteOffsetPosition End;
};
class SyntaxParsingCache {

244
include/swift/Syntax/AbsoluteRawSyntax.h Normal file
View File

@@ -0,0 +1,244 @@
//===--- AbsoluteRawSyntax.h ------------------------------------*- C++ -*-===//
//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See https://swift.org/LICENSE.txt for license information
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
//
//===----------------------------------------------------------------------===//
#ifndef SWIFT_SYNTAX_ABSOLUTERAWSYNTAX_H
#define SWIFT_SYNTAX_ABSOLUTERAWSYNTAX_H
#include "swift/Basic/SourceLoc.h"
#include "swift/Syntax/RawSyntax.h"
namespace swift {
namespace syntax {
/// Type-safe wrapper around a \c size_t that describes a node's index in its
/// tree. This represents the index at which the node will be encountered during
/// a depth-first traversal of the tree.
class SyntaxIndexInTree {
size_t IndexInTree;
explicit SyntaxIndexInTree(size_t IndexInTree) : IndexInTree(IndexInTree) {}
public:
static SyntaxIndexInTree zero() { return SyntaxIndexInTree(0); }
/// Assuming that this index points to the start of \p Raw, advance it so that
/// it points to the next sibling of \p Raw.
SyntaxIndexInTree advancedBy(const RC<RawSyntax> &Raw) const;
/// Assuming that this index points to the next sibling of \p Raw, reverse it
/// so that it points to the start of \p Raw.
SyntaxIndexInTree reversedBy(const RC<RawSyntax> &Raw) const;
/// Advance this index to point to its first immediate child.
SyntaxIndexInTree advancedToFirstChild() const;
};
/// A syntax identifier that globally identifies a \c Syntax node.
/// In contrast to \c NodeId of \c RawSyntax, this also takes into account the
/// node's position in the tree. For example, a source file may contain two
/// int literals with value 0. The corresponding \c RawSyntax nodes can share
/// the same \c NodeId. However, because the literals occur in different
/// locations of the syntax tree, their \c SyntaxIdentifiers are different.
class SyntaxIdentifier {
public:
using RootIdType = size_t;
private:
static std::atomic<RootIdType> NextUnusedRootId;
/// An integer that identifies the tree in which the node represented by this
/// identifier lives.
RootIdType RootId;
/// The position of the node represented by this identifier in the syntax
/// tree.
SyntaxIndexInTree IndexInTree;
public:
SyntaxIdentifier(RootIdType RootId, SyntaxIndexInTree IndexInTree)
: RootId(RootId), IndexInTree(IndexInTree) {
assert(RootId < NextUnusedRootId && "New RootIds should only be created "
"using the newRoot constructor.");
}
/// Create a SyntaxIdentifier that refers to the root of a new syntax tree.
static SyntaxIdentifier newRoot() {
return SyntaxIdentifier(NextUnusedRootId++, SyntaxIndexInTree::zero());
}
RootIdType getRootId() const { return RootId; }
SyntaxIndexInTree getIndexInTree() const { return IndexInTree; }
/// Assuming that this identifier points to the start of \p Raw, advance it so
/// that it points to the next sibling of \p Raw.
SyntaxIdentifier advancedBy(const RC<RawSyntax> &Raw) const {
auto NewIndexInTree = IndexInTree.advancedBy(Raw);
return SyntaxIdentifier(RootId, NewIndexInTree);
}
/// Assuming that this identifier points to the next sibling of \p Raw,
/// reverse it so that it points to the start of \p Raw.
SyntaxIdentifier reversedBy(const RC<RawSyntax> &Raw) const {
auto NewIndexInTree = IndexInTree.reversedBy(Raw);
return SyntaxIdentifier(RootId, NewIndexInTree);
}
/// Get the identifier of the first immediate child.
SyntaxIdentifier advancedToFirstChild() const {
auto NewIndexInTree = IndexInTree.advancedToFirstChild();
return SyntaxIdentifier(RootId, NewIndexInTree);
}
};
/// Represents a node's position in a syntax tree, described by its overal
/// textual offset and the position within its parent.
class AbsoluteSyntaxPosition {
public:
using OffsetType = uint32_t;
using IndexInParentType = uint32_t;
private:
/// The text offset where this node starts within its syntax tree, counted in
/// UTF-8 bytes.
OffsetType Offset;
/// The node's index within its parent, i.e.
/// `node.parent.childAt(IndexInParent) = node`.
IndexInParentType IndexInParent;
public:
AbsoluteSyntaxPosition(OffsetType Offset, IndexInParentType IndexInParent)
: Offset(Offset), IndexInParent(IndexInParent) {}
/// Create a new \c AbsoluteSyntaxPosition that refers to the root of a syntax
/// tree.
static AbsoluteSyntaxPosition forRoot() {
return AbsoluteSyntaxPosition(0, 0);
}
OffsetType getOffset() const { return Offset; }
IndexInParentType getIndexInParent() const { return IndexInParent; }
/// Assuming that this position points to the start of \p Raw, advance it so
/// that it points to the next sibling of \p Raw.
AbsoluteSyntaxPosition advancedBy(const RC<RawSyntax> &Raw) const;
/// Assuming that this position points to the next sibling of \p Raw, reverse
/// it so that it points to the start of \p Raw.
AbsoluteSyntaxPosition reversedBy(const RC<RawSyntax> &Raw) const;
/// Get the position of the node's first immediate child.
AbsoluteSyntaxPosition advancedToFirstChild() const {
return AbsoluteSyntaxPosition(Offset, 0);
}
};
/// A type-safe wrapper that describes a node's textual position within a source
/// file, represented by its UTF-8 byte offset from the start.
class AbsoluteOffsetPosition {
AbsoluteSyntaxPosition::OffsetType Offset;
public:
explicit AbsoluteOffsetPosition(AbsoluteSyntaxPosition::OffsetType Offset)
: Offset(Offset) {}
AbsoluteOffsetPosition(AbsoluteSyntaxPosition Position)
: Offset(Position.getOffset()) {}
AbsoluteSyntaxPosition::OffsetType getOffset() const { return Offset; }
/// Return a position that has been advanced by \p Advance UTF-8 bytes.s
AbsoluteOffsetPosition advancedBy(int Advance) {
return AbsoluteOffsetPosition(Offset + Advance);
}
};
/// Various information that enrich a \c RawSyntax node with information on how
/// it's located within the syntax tree.
class AbsoluteSyntaxInfo {
AbsoluteSyntaxPosition Position;
SyntaxIdentifier NodeId;
public:
AbsoluteSyntaxInfo(AbsoluteSyntaxPosition Position, SyntaxIdentifier NodeId)
: Position(Position), NodeId(NodeId) {}
static AbsoluteSyntaxInfo forRoot() {
return AbsoluteSyntaxInfo(AbsoluteSyntaxPosition::forRoot(),
SyntaxIdentifier::newRoot());
}
AbsoluteSyntaxPosition getPosition() const { return Position; }
SyntaxIdentifier getNodeId() const { return NodeId; }
/// Assuming that this info points to the start of \p Raw, advance it so
/// that it points to the next sibling of \p Raw.
AbsoluteSyntaxInfo advancedBy(const RC<RawSyntax> &Raw) const {
auto NewNodeId = NodeId.advancedBy(Raw);
auto NewPosition = Position.advancedBy(Raw);
return AbsoluteSyntaxInfo(NewPosition, NewNodeId);
}
/// Assuming that this info points to the next sibling of \p Raw, reverse
/// it so that it points to the start of \p Raw.
AbsoluteSyntaxInfo reversedBy(const RC<RawSyntax> &Raw) const {
auto NewNodeId = NodeId.reversedBy(Raw);
auto NewPosition = Position.reversedBy(Raw);
return AbsoluteSyntaxInfo(NewPosition, NewNodeId);
}
/// Get the information of the node's first immediate child.
AbsoluteSyntaxInfo advancedToFirstChild() const {
auto NewNodeId = NodeId.advancedToFirstChild();
auto NewPosition = Position.advancedToFirstChild();
return AbsoluteSyntaxInfo(NewPosition, NewNodeId);
}
};
/// A \c RawSyntax node that is enrichted with information of its position
/// within the syntax tree it lives in.
struct AbsoluteRawSyntax {
const RC<RawSyntax> Raw;
const AbsoluteSyntaxInfo Info;
public:
AbsoluteRawSyntax(const RC<RawSyntax> &Raw, AbsoluteSyntaxInfo Info)
: Raw(Raw), Info(Info) {}
/// Construct a \c AbsoluteRawSyntax for a \c RawSyntax node that represents
/// the syntax tree's root.
static AbsoluteRawSyntax forRoot(const RC<RawSyntax> &Raw) {
return AbsoluteRawSyntax(Raw, AbsoluteSyntaxInfo::forRoot());
}
const RC<RawSyntax> &getRaw() const { return Raw; }
AbsoluteSyntaxInfo getInfo() const { return Info; }
/// Get the position at which the leading triva of this node starts.
AbsoluteSyntaxPosition getPosition() const { return Info.getPosition(); };
SyntaxIdentifier getNodeId() const { return Info.getNodeId(); };
AbsoluteSyntaxPosition::IndexInParentType getIndexInParent() const {
return getPosition().getIndexInParent();
}
};
} // end namespace syntax
} // end namespace swift
namespace llvm {
raw_ostream &operator<<(raw_ostream &OS,
swift::syntax::AbsoluteOffsetPosition Pos);
} // end namespace llvm
#endif // SWIFT_SYNTAX_ABSOLUTERAWSYNTAX_H

270
include/swift/Syntax/RawSyntax.h
View File

@@ -126,78 +126,6 @@ template <typename CursorType> constexpr CursorIndex cursorIndex(CursorType C) {
return static_cast<CursorIndex>(C);
}
/// An absolute position in a source file as text - the absolute offset from
/// the start, line, and column.
class AbsolutePosition {
uintptr_t Offset = 0;
uint32_t Line = 1;
uint32_t Column = 1;
public:
/// Add some number of columns to the position.
void addColumns(uint32_t Columns) {
Column += Columns;
Offset += Columns;
}
/// Add some number of newlines to the position, resetting the column.
/// Size is byte size of newline char.
/// '\n' and '\r' are 1, '\r\n' is 2.
void addNewlines(uint32_t NewLines, uint32_t Size) {
Line += NewLines;
Column = 1;
Offset += NewLines * Size;
}
/// Use some text as a reference for adding to the absolute position,
/// taking note of newlines, etc.
void addText(StringRef Str) {
const char * C = Str.begin();
while (C != Str.end()) {
switch (*C++) {
case '\n':
addNewlines(1, 1);
break;
case '\r':
if (C != Str.end() && *C == '\n') {
addNewlines(1, 2);
++C;
} else {
addNewlines(1, 1);
}
break;
default:
addColumns(1);
break;
}
}
}
/// Get the line number of this position.
uint32_t getLine() const { return Line; }
/// Get the column number of this position.
uint32_t getColumn() const { return Column; }
/// Get the line and column number of this position.
std::pair<uint32_t, uint32_t> getLineAndColumn() const {
return {Line, Column};
}
/// Get the absolute offset of this position, suitable for indexing into a
/// buffer if applicable.
uintptr_t getOffset() const { return Offset; }
/// Print the line and column as "l:c" to the given output stream.
void printLineAndColumn(llvm::raw_ostream &OS) const;
/// Dump a description of this position to the given output stream
/// for debugging.
void dump(llvm::raw_ostream &OS) const;
SWIFT_DEBUG_DUMP;
};
/// An indicator of whether a Syntax node was found or written in the source.
///
/// This is not an 'implicit' bit.
@@ -239,33 +167,35 @@ class RawSyntax final
RC<SyntaxArena> Arena;
union {
uint64_t OpaqueBits;
struct {
/// The kind of syntax this node represents.
unsigned Kind : bitmax(NumSyntaxKindBits, 8);
// FIXME: Reduce TextLength to 30 bits so that common fits in 4 bytes?
/// Number of bytes this node takes up spelled out in the source code
unsigned TextLength : 32;
/// Whether this piece of syntax was actually present in the source.
unsigned Presence : 1;
unsigned IsToken : 1;
} Common;
enum { NumRawSyntaxBits = bitmax(NumSyntaxKindBits, 8) + 1 };
enum { NumRawSyntaxBits = 32 + 1 + 1 };
// For "layout" nodes.
struct {
static_assert(NumRawSyntaxBits <= 32,
"Only 32 bits reserved for standard syntax bits");
uint64_t : bitmax(NumRawSyntaxBits, 32); // align to 32 bits
static_assert(NumRawSyntaxBits <= 64,
"Only 64 bits reserved for standard syntax bits");
uint64_t : bitmax(NumRawSyntaxBits, 64); // align to 32 bits
/// Number of children this "layout" node has.
unsigned NumChildren : 32;
/// Number of bytes this node takes up spelled out in the source code
/// A value of UINT32_MAX indicates that the text length has not been
/// computed yet.
unsigned TextLength : 32;
/// Total number of sub nodes, i.e. number of transitive children of this
/// node. This does not include the node itself.
unsigned TotalSubNodeCount : 32;
/// The kind of syntax this node represents.
unsigned Kind : bitmax(NumSyntaxKindBits, 8);
} Layout;
// For "token" nodes.
struct {
static_assert(NumRawSyntaxBits <= 16,
"Only 16 bits reserved for standard syntax bits");
uint64_t : bitmax(NumRawSyntaxBits, 16); // align to 16 bits
static_assert(NumRawSyntaxBits <= 64,
"Only 64 bits reserved for standard syntax bits");
uint64_t : bitmax(NumRawSyntaxBits, 64); // align to 16 bits
/// The kind of token this "token" node represents.
unsigned TokenKind : 16;
/// Number of leading trivia pieces.
@@ -293,7 +223,7 @@ class RawSyntax final
/// underlying storage.
/// If \p NodeId is \c None, the next free NodeId is used, if it is passed,
/// the caller needs to assure that the node ID has not been used yet.
RawSyntax(SyntaxKind Kind, ArrayRef<RC<RawSyntax>> Layout,
RawSyntax(SyntaxKind Kind, ArrayRef<RC<RawSyntax>> Layout, size_t TextLength,
SourcePresence Presence, const RC<SyntaxArena> &Arena,
llvm::Optional<SyntaxNodeId> NodeId);
/// Constructor for creating token nodes
@@ -301,7 +231,8 @@ class RawSyntax final
/// underlying storage.
/// If \p NodeId is \c None, the next free NodeId is used, if it is passed,
/// the caller needs to assure that the NodeId has not been used yet.
RawSyntax(tok TokKind, OwnedString Text, ArrayRef<TriviaPiece> LeadingTrivia,
RawSyntax(tok TokKind, OwnedString Text, size_t TextLength,
ArrayRef<TriviaPiece> LeadingTrivia,
ArrayRef<TriviaPiece> TrailingTrivia, SourcePresence Presence,
const RC<SyntaxArena> &Arena, llvm::Optional<SyntaxNodeId> NodeId);
@@ -348,47 +279,93 @@ public:
/// Make a raw "layout" syntax node.
static RC<RawSyntax> make(SyntaxKind Kind, ArrayRef<RC<RawSyntax>> Layout,
SourcePresence Presence,
size_t TextLength, SourcePresence Presence,
const RC<SyntaxArena> &Arena,
llvm::Optional<SyntaxNodeId> NodeId = llvm::None);
static RC<RawSyntax> make(SyntaxKind Kind, ArrayRef<RC<RawSyntax>> Layout,
size_t TextLength, SourcePresence Presence,
llvm::Optional<SyntaxNodeId> NodeId = llvm::None) {
RC<SyntaxArena> Arena = nullptr;
return make(Kind, Layout, Presence, Arena, NodeId);
return make(Kind, Layout, TextLength, Presence, /*Arena=*/nullptr, NodeId);
}
/// Make a raw "layout" syntax node that was allocated in \p Arena.
static RC<RawSyntax> make(SyntaxKind Kind, ArrayRef<RC<RawSyntax>> Layout,
static RC<RawSyntax>
makeAndCalcLength(SyntaxKind Kind, ArrayRef<RC<RawSyntax>> Layout,
SourcePresence Presence, const RC<SyntaxArena> &Arena,
llvm::Optional<SyntaxNodeId> NodeId = llvm::None) {
size_t TextLength = 0;
for (auto Child : Layout) {
if (Child) {
TextLength += Child->getTextLength();
}
}
return make(Kind, Layout, TextLength, Presence, Arena, NodeId);
}
static RC<RawSyntax>
makeAndCalcLength(SyntaxKind Kind, ArrayRef<RC<RawSyntax>> Layout,
SourcePresence Presence,
llvm::Optional<SyntaxNodeId> NodeId = llvm::None) {
return makeAndCalcLength(Kind, Layout, Presence, /*Arena=*/nullptr, NodeId);
}
/// Make a raw "token" syntax node.
static RC<RawSyntax> make(tok TokKind, OwnedString Text, size_t TextLength,
ArrayRef<TriviaPiece> LeadingTrivia,
ArrayRef<TriviaPiece> TrailingTrivia,
SourcePresence Presence,
const RC<SyntaxArena> &Arena,
llvm::Optional<SyntaxNodeId> NodeId = llvm::None);
/// Make a raw "token" syntax node.
static RC<RawSyntax> make(tok TokKind, OwnedString Text,
static RC<RawSyntax> make(tok TokKind, OwnedString Text, size_t TextLength,
ArrayRef<TriviaPiece> LeadingTrivia,
ArrayRef<TriviaPiece> TrailingTrivia,
SourcePresence Presence,
llvm::Optional<SyntaxNodeId> NodeId = llvm::None) {
RC<SyntaxArena> Arena = nullptr;
return make(TokKind, Text, LeadingTrivia, TrailingTrivia, Presence, Arena,
NodeId);
return make(TokKind, Text, TextLength, LeadingTrivia, TrailingTrivia,
Presence, /*Arena=*/nullptr, NodeId);
}
/// Make a raw "token" syntax node that was allocated in \p Arena.
static RC<RawSyntax> make(tok TokKind, OwnedString Text,
ArrayRef<TriviaPiece> LeadingTrivia,
ArrayRef<TriviaPiece> TrailingTrivia,
SourcePresence Presence,
const RC<SyntaxArena> &Arena,
llvm::Optional<SyntaxNodeId> NodeId = llvm::None);
static RC<RawSyntax>
makeAndCalcLength(tok TokKind, OwnedString Text,
ArrayRef<TriviaPiece> LeadingTrivia,
ArrayRef<TriviaPiece> TrailingTrivia,
SourcePresence Presence, const RC<SyntaxArena> &Arena,
llvm::Optional<SyntaxNodeId> NodeId = llvm::None) {
size_t TextLength = 0;
if (Presence != SourcePresence::Missing) {
for (auto Trivia : LeadingTrivia) {
TextLength += Trivia.getTextLength();
}
TextLength += Text.size();
for (auto Trivia : TrailingTrivia) {
TextLength += Trivia.getTextLength();
}
}
return make(TokKind, Text, TextLength, LeadingTrivia, TrailingTrivia,
Presence, Arena, NodeId);
}
static RC<RawSyntax> makeAndCalcLength(
tok TokKind, OwnedString Text, ArrayRef<TriviaPiece> LeadingTrivia,
ArrayRef<TriviaPiece> TrailingTrivia, SourcePresence Presence,
llvm::Optional<SyntaxNodeId> NodeId = llvm::None) {
return makeAndCalcLength(TokKind, Text, LeadingTrivia, TrailingTrivia,
Presence, /*Arena=*/nullptr, NodeId);
}
/// Make a missing raw "layout" syntax node.
static RC<RawSyntax> missing(SyntaxKind Kind,
RC<SyntaxArena> Arena = nullptr) {
return make(Kind, {}, SourcePresence::Missing, Arena);
return make(Kind, {}, /*TextLength=*/0, SourcePresence::Missing, Arena);
}
/// Make a missing raw "token" syntax node.
static RC<RawSyntax> missing(tok TokKind, OwnedString Text,
RC<SyntaxArena> Arena = nullptr) {
return make(TokKind, Text, {}, {}, SourcePresence::Missing, Arena);
return make(TokKind, Text, /*TextLength=*/0, {}, {},
SourcePresence::Missing, Arena);
}
/// @}
@@ -398,7 +375,25 @@ public:
}
SyntaxKind getKind() const {
return static_cast<SyntaxKind>(Bits.Common.Kind);
if (Bits.Common.IsToken) {
return SyntaxKind::Token;
} else {
return static_cast<SyntaxKind>(Bits.Layout.Kind);
}
}
/// Get the number of nodes included in the subtree spanned by this node.
/// This includes all transitive children and this node itself.
size_t getTotalNodes() { return getTotalSubNodeCount() + 1; }
/// Get the number of transitive children of this node. This does not include
/// the node itself.
size_t getTotalSubNodeCount() {
if (isToken()) {
return 0;
} else {
return Bits.Layout.TotalSubNodeCount;
}
}
/// Get an ID for this node that is stable across incremental parses
@@ -430,7 +425,7 @@ public:
bool isUnknown() const { return isUnknownKind(getKind()); }
/// Return true if this raw syntax node is a token.
bool isToken() const { return isTokenKind(getKind()); }
bool isToken() const { return Bits.Common.IsToken; }
/// \name Getter routines for SyntaxKind::Token.
/// @{
@@ -476,8 +471,9 @@ public:
/// trivia instead.
RC<RawSyntax>
withLeadingTrivia(ArrayRef<TriviaPiece> NewLeadingTrivia) const {
return make(getTokenKind(), getOwnedTokenText(), NewLeadingTrivia,
getTrailingTrivia(), getPresence());
return makeAndCalcLength(getTokenKind(), getOwnedTokenText(),
NewLeadingTrivia, getTrailingTrivia(),
getPresence());
}
RC<RawSyntax> withLeadingTrivia(Trivia NewLeadingTrivia) const {
@@ -488,8 +484,9 @@ public:
/// trivia instead.
RC<RawSyntax>
withTrailingTrivia(ArrayRef<TriviaPiece> NewTrailingTrivia) const {
return make(getTokenKind(), getOwnedTokenText(), getLeadingTrivia(),
NewTrailingTrivia, getPresence());
return makeAndCalcLength(getTokenKind(), getOwnedTokenText(),
getLeadingTrivia(), NewTrailingTrivia,
getPresence());
}
RC<RawSyntax> withTrailingTrivia(Trivia NewTrailingTrivia) const {
@@ -521,23 +518,7 @@ public:
}
/// Return the number of bytes this node takes when spelled out in the source
size_t getTextLength() {
// For tokens the computation of the length is fast enough to justify the
// space for caching it. For layout nodes, we cache the length to avoid
// traversing the tree
// FIXME: Or would it be sensible to cache the size of token nodes as well?
if (isToken()) {
AbsolutePosition Pos;
accumulateAbsolutePosition(Pos);
return Pos.getOffset();
} else {
if (Bits.Layout.TextLength == UINT32_MAX) {
Bits.Layout.TextLength = computeTextLength();
}
return Bits.Layout.TextLength;
}
}
size_t getTextLength() { return Bits.Common.TextLength; }
/// @}
@@ -550,22 +531,26 @@ public:
/// Return a new raw syntax node with the given new layout element replacing
/// another at some cursor position.
RC<RawSyntax>
replaceChild(CursorIndex Index, RC<RawSyntax> NewLayoutElement) const;
RC<RawSyntax> replacingChild(CursorIndex Index,
RC<RawSyntax> NewLayoutElement) const;
/// @}
/// Advance the provided AbsolutePosition by the full width of this node.
///
/// If this is token node, returns the AbsolutePosition of the start of the
/// token's nontrivial text. Otherwise, return the position of the first
/// token. If this contains no tokens, return None.
llvm::Optional<AbsolutePosition>
accumulateAbsolutePosition(AbsolutePosition &Pos) const;
size_t getLeadingTriviaLength() {
size_t Length = 0;
for (auto Trivia : getLeadingTrivia()) {
Length += Trivia.getTextLength();
}
return Length;
}
/// Advance the provided AbsolutePosition by the first trivia of this node.
/// Return true if we found this trivia; otherwise false.
bool accumulateLeadingTrivia(AbsolutePosition &Pos) const;
size_t getTrailingTriviaLength() {
size_t Length = 0;
for (auto Trivia : getTrailingTrivia()) {
Length += Trivia.getTextLength();
}
return Length;
}
/// Print this piece of syntax recursively.
void print(llvm::raw_ostream &OS, SyntaxPrintOptions Opts) const;
@@ -584,8 +569,5 @@ public:
} // end namespace syntax
} // end namespace swift
namespace llvm {
raw_ostream &operator<<(raw_ostream &OS, swift::syntax::AbsolutePosition Pos);
} // end namespace llvm
#endif // SWIFT_SYNTAX_RAWSYNTAX_H

8
include/swift/Syntax/Serialization/SyntaxDeserialization.h
View File

@@ -162,7 +162,7 @@ template <> struct MappingTraits<swift::RC<swift::RawSyntax>> {
StringRef nodeIdString;
in.mapRequired("id", nodeIdString);
unsigned nodeId = std::atoi(nodeIdString.data());
value = swift::RawSyntax::make(
value = swift::RawSyntax::makeAndCalcLength(
tokenKind, swift::OwnedString::makeRefCounted(text), leadingTrivia,
trailingTrivia, presence, /*Arena=*/nullptr, nodeId);
} else {
@@ -178,8 +178,8 @@ template <> struct MappingTraits<swift::RC<swift::RawSyntax>> {
StringRef nodeIdString;
in.mapRequired("id", nodeIdString);
unsigned nodeId = std::atoi(nodeIdString.data());
value = swift::RawSyntax::make(kind, layout, presence, /*Arena=*/nullptr,
nodeId);
value = swift::RawSyntax::makeAndCalcLength(kind, layout, presence,
/*Arena=*/nullptr, nodeId);
}
}
};
@@ -198,7 +198,7 @@ public:
llvm::Optional<swift::SourceFileSyntax> getSourceFileSyntax() {
swift::RC<swift::RawSyntax> raw;
Input >> raw;
return swift::make<swift::SourceFileSyntax>(raw);
return swift::makeRoot<swift::SourceFileSyntax>(raw);
}
};
} // namespace json

40
include/swift/Syntax/Syntax.h
View File

@@ -39,9 +39,8 @@ namespace syntax {
struct SyntaxVisitor;
class SourceFileSyntax;
template <typename SyntaxNode>
SyntaxNode make(RC<RawSyntax> Raw) {
auto Data = SyntaxData::make(Raw);
template <typename SyntaxNode> SyntaxNode makeRoot(RC<RawSyntax> Raw) {
auto Data = SyntaxData::make(AbsoluteRawSyntax::forRoot(Raw));
return { Data, Data.get() };
}
@@ -80,7 +79,7 @@ public:
SyntaxKind getKind() const;
/// Get the shared raw syntax.
const RC<RawSyntax> &getRaw() const;
const RC<RawSyntax> getRaw() const;
/// Get an ID for this node that is stable across incremental parses
SyntaxNodeId getId() const { return getRaw()->getId(); }
@@ -192,23 +191,32 @@ public:
/// Recursively visit this node.
void accept(SyntaxVisitor &Visitor);
/// Get the absolute position of this raw syntax: its offset, line,
/// and column.
AbsolutePosition getAbsolutePosition() const {
return Data->getAbsolutePosition();
/// Same as \c getAbsolutePositionAfterLeadingTrivia.
AbsoluteOffsetPosition getAbsolutePosition() const {
return getAbsolutePositionAfterLeadingTrivia();
}
/// Get the absolute end position (exclusively) where the trailing trivia of
/// this node ends.
AbsolutePosition getAbsoluteEndPositionAfterTrailingTrivia() const {
return Data->getAbsoluteEndPositionAfterTrailingTrivia();
}
/// Get the absolute position at which the leading trivia of this node starts.
AbsolutePosition getAbsolutePositionBeforeLeadingTrivia() const {
/// Get the offset at which the leading trivia of this node starts.
AbsoluteOffsetPosition getAbsolutePositionBeforeLeadingTrivia() const {
return Data->getAbsolutePositionBeforeLeadingTrivia();
}
/// Get the offset at which the actual content (i.e. non-triva) of this node
/// starts.
AbsoluteOffsetPosition getAbsolutePositionAfterLeadingTrivia() const {
return Data->getAbsolutePositionAfterLeadingTrivia();
}
/// Get the offset at which the trailing trivia of this node starts.
AbsoluteOffsetPosition getAbsoluteEndPositionBeforeTrailingTrivia() const {
return Data->getAbsoluteEndPositionBeforeTrailingTrivia();
}
/// Get the offset at which the trailing trivia of this node starts.
AbsoluteOffsetPosition getAbsoluteEndPositionAfterTrailingTrivia() const {
return Data->getAbsoluteEndPositionAfterTrailingTrivia();
}
// TODO: hasSameStructureAs ?
};

20
include/swift/Syntax/SyntaxCollection.h
View File

@@ -63,9 +63,9 @@ private:
List.reserve(Elements.size());
for (auto &Elt : Elements)
List.push_back(Elt.getRaw());
auto Raw = RawSyntax::make(CollectionKind, List,
SourcePresence::Present);
return SyntaxData::make(Raw);
auto Raw = RawSyntax::makeAndCalcLength(CollectionKind, List,
SourcePresence::Present);
return SyntaxData::make(AbsoluteRawSyntax::forRoot(Raw));
}
SyntaxCollection(const RC<SyntaxData> Root): Syntax(Root, Root.get()) {}
@@ -120,7 +120,7 @@ public:
std::copy(OldLayout.begin(), OldLayout.end(), back_inserter(NewLayout));
NewLayout.push_back(E.getRaw());
auto Raw = RawSyntax::make(CollectionKind, NewLayout, getRaw()->getPresence());
return Data->replaceSelf<SyntaxCollection<CollectionKind, Element>>(Raw);
return Data->replacingSelf<SyntaxCollection<CollectionKind, Element>>(Raw);
}
/// Return a new collection with an element removed from the end.
@@ -130,7 +130,7 @@ public:
assert(!empty());
auto NewLayout = getRaw()->getLayout().drop_back();
auto Raw = RawSyntax::make(CollectionKind, NewLayout, getRaw()->getPresence());
return Data->replaceSelf<SyntaxCollection<CollectionKind, Element>>(Raw);
return Data->replacingSelf<SyntaxCollection<CollectionKind, Element>>(Raw);
}
/// Return a new collection with the given element appended to the front.
@@ -141,7 +141,7 @@ public:
std::copy(OldLayout.begin(), OldLayout.end(),
std::back_inserter(NewLayout));
auto Raw = RawSyntax::make(CollectionKind, NewLayout, getRaw()->getPresence());
return Data->replaceSelf<SyntaxCollection<CollectionKind, Element>>(Raw);
return Data->replacingSelf<SyntaxCollection<CollectionKind, Element>>(Raw);
}
/// Return a new collection with an element removed from the end.
@@ -151,7 +151,7 @@ public:
assert(!empty());
auto NewLayout = getRaw()->getLayout().drop_front();
auto Raw = RawSyntax::make(CollectionKind, NewLayout, getRaw()->getPresence());
return Data->replaceSelf<SyntaxCollection<CollectionKind, Element>>(Raw);
return Data->replacingSelf<SyntaxCollection<CollectionKind, Element>>(Raw);
}
/// Return a new collection with the Element inserted at index i.
@@ -170,7 +170,7 @@ public:
std::copy(OldLayout.begin() + i, OldLayout.end(),
std::back_inserter(NewLayout));
auto Raw = RawSyntax::make(CollectionKind, NewLayout, getRaw()->getPresence());
return Data->replaceSelf<SyntaxCollection<CollectionKind, Element>>(Raw);
return Data->replacingSelf<SyntaxCollection<CollectionKind, Element>>(Raw);
}
/// Return a new collection with the element removed at index i.
@@ -181,13 +181,13 @@ public:
std::advance(iterator, i);
NewLayout.erase(iterator);
auto Raw = RawSyntax::make(CollectionKind, NewLayout, getRaw()->getPresence());
return Data->replaceSelf<SyntaxCollection<CollectionKind, Element>>(Raw);
return Data->replacingSelf<SyntaxCollection<CollectionKind, Element>>(Raw);
}
/// Return an empty syntax collection of this type.
SyntaxCollection<CollectionKind, Element> cleared() const {
auto Raw = RawSyntax::make(CollectionKind, {}, getRaw()->getPresence());
return Data->replaceSelf<SyntaxCollection<CollectionKind, Element>>(Raw);
return Data->replacingSelf<SyntaxCollection<CollectionKind, Element>>(Raw);
}
static bool kindof(SyntaxKind Kind) {

164
include/swift/Syntax/SyntaxData.h
View File

@@ -40,6 +40,7 @@
#define SWIFT_SYNTAX_SYNTAXDATA_H
#include "swift/Basic/Debug.h"
#include "swift/Syntax/AbsoluteRawSyntax.h"
#include "swift/Syntax/AtomicCache.h"
#include "swift/Syntax/RawSyntax.h"
#include "swift/Syntax/References.h"
@@ -55,111 +56,72 @@ namespace syntax {
/// This structure should not contain significant public
/// API or internal modification API.
///
/// This is only for holding a strong reference to the RawSyntax, a weak
/// reference to the parent, and, in subclasses, lazily created strong
/// references to non-terminal child nodes.
class SyntaxData final
: public llvm::ThreadSafeRefCountedBase<SyntaxData>,
private llvm::TrailingObjects<SyntaxData, AtomicCache<SyntaxData>> {
friend TrailingObjects;
/// It is essentially a wrapper around \c AbsoluteRawSyntax that also keeps
/// track of the parent.
class SyntaxData final : public llvm::ThreadSafeRefCountedBase<SyntaxData> {
using RootDataPair = std::pair<RC<SyntaxData>, RC<SyntaxData>>;
/// The shared raw syntax representing this syntax data node.
const RC<RawSyntax> Raw;
const AbsoluteRawSyntax AbsoluteRaw;
const RC<SyntaxData> Parent;
/// The parent of this syntax.
///
/// WARNING! Do not access this directly. Use getParent(),
/// which enforces nullptr checking.
const SyntaxData *Parent;
SyntaxData(AbsoluteRawSyntax AbsoluteRaw, const RC<SyntaxData> &Parent)
: AbsoluteRaw(AbsoluteRaw), Parent(Parent) {}
/// The index into the parent's child layout.
///
/// If there is no parent, this is 0.
const CursorIndex IndexInParent;
/// Cache the absolute position of this node.
Optional<AbsolutePosition> PositionCache;
size_t numTrailingObjects(OverloadToken<AtomicCache<SyntaxData>>) const {
return Raw->getNumChildren();
}
SyntaxData(RC<RawSyntax> Raw, const SyntaxData *Parent = nullptr,
CursorIndex IndexInParent = 0)
: Raw(Raw), Parent(Parent), IndexInParent(IndexInParent) {
auto *I = getTrailingObjects<AtomicCache<SyntaxData>>();
for (auto *E = I + getNumChildren(); I != E; ++I)
::new (static_cast<void *>(I)) AtomicCache<SyntaxData>();
}
/// With a new RawSyntax node, create a new node from this one and
/// With a new \c RawSyntax node, create a new node from this one and
/// recursively rebuild the parental chain up to the root.
///
/// DO NOT expose this as public API.
RootDataPair replaceSelf(const RC<RawSyntax> NewRaw) const {
RootDataPair replacingSelf(const RC<RawSyntax> &NewRaw) const {
if (hasParent()) {
auto NewRootAndParent = Parent->replaceChild(NewRaw, IndexInParent);
auto NewMe = NewRootAndParent.second->getChild(IndexInParent);
auto NewRootAndParent =
Parent->replacingChild(NewRaw, getIndexInParent());
auto NewMe = NewRootAndParent.second->getChild(getIndexInParent());
return { NewRootAndParent.first, NewMe.get() };
} else {
auto NewMe = make(NewRaw, nullptr, IndexInParent);
auto NewMe = make(AbsoluteRawSyntax::forRoot(NewRaw), nullptr);
return { NewMe, NewMe.get() };
}
}
/// Create the data for a child node with the raw syntax in our layout
/// at the provided index.
/// DO NOT expose this as public API.
RC<SyntaxData> realizeSyntaxNode(CursorIndex Index) const {
if (auto &RawChild = Raw->getChild(Index))
return SyntaxData::make(RawChild, this, Index);
return nullptr;
}
/// Replace a child in the raw syntax and recursively rebuild the
/// parental chain up to the root.
///
/// DO NOT expose this as public API.
template <typename CursorType>
RootDataPair replaceChild(const RC<RawSyntax> RawChild,
CursorType ChildCursor) const {
auto NewRaw = Raw->replaceChild(ChildCursor, RawChild);
return replaceSelf(NewRaw);
RootDataPair replacingChild(const RC<RawSyntax> &RawChild,
CursorType ChildCursor) const {
auto NewRaw = AbsoluteRaw.getRaw()->replacingChild(ChildCursor, RawChild);
return replacingSelf(NewRaw);
}
ArrayRef<AtomicCache<SyntaxData>> getChildren() const {
return {getTrailingObjects<AtomicCache<SyntaxData>>(), getNumChildren()};
RC<SyntaxData> getRefCountedThis() const {
// FIXME: Is the usage of const_cast safe here?
return RC<SyntaxData>(const_cast<SyntaxData *>(this));
}
public:
/// Disable sized deallocation for SyntaxData, because it has tail-allocated
/// data.
void operator delete(void *p) { ::operator delete(p); }
/// Get the node immediately before this current node that does contain a
/// non-missing token. Return nullptr if we cannot find such node.
/// non-missing token. Return \c nullptr if we cannot find such node.
RC<SyntaxData> getPreviousNode() const;
/// Get the node immediately after this current node that does contain a
/// non-missing token. Return nullptr if we cannot find such node.
/// non-missing token. Return \c nullptr if we cannot find such node.
RC<SyntaxData> getNextNode() const;
/// Get the first non-missing token node in this tree. Return nullptr if this
/// node does not contain non-missing tokens.
/// Get the first non-missing token node in this tree. Return \c nullptr if
/// this node does not contain non-missing tokens.
RC<SyntaxData> getFirstToken() const;
~SyntaxData() {
for (auto &I : getChildren())
I.~AtomicCache<SyntaxData>();
}
/// Get the last non-missing token node in this tree. Return \c nullptr if
/// this node does not contain non-missing tokens.
RC<SyntaxData> getLastToken() const;
/// Constructs a SyntaxNode by replacing `self` and recursively building
/// the parent chain up to the root.
template <typename SyntaxNode>
SyntaxNode replaceSelf(const RC<RawSyntax> NewRaw) const {
auto NewRootAndData = replaceSelf(NewRaw);
SyntaxNode replacingSelf(const RC<RawSyntax> &NewRaw) const {
auto NewRootAndData = replacingSelf(NewRaw);
return { NewRootAndData.first, NewRootAndData.second.get() };
}
@@ -168,34 +130,30 @@ public:
///
/// DO NOT expose this as public API.
template <typename SyntaxNode, typename CursorType>
SyntaxNode replaceChild(const RC<RawSyntax> RawChild,
CursorType ChildCursor) const {
auto NewRootAndParent = replaceChild(RawChild, ChildCursor);
SyntaxNode replacingChild(const RC<RawSyntax> &RawChild,
CursorType ChildCursor) const {
auto NewRootAndParent = replacingChild(RawChild, ChildCursor);
return SyntaxNode {
NewRootAndParent.first,
NewRootAndParent.second.get()
};
}
static RC<SyntaxData> make(RC<RawSyntax> Raw,
const SyntaxData *Parent = nullptr,
CursorIndex IndexInParent = 0);
/// Make a new \c SyntaxData node for the tree's root.
static RC<SyntaxData> make(AbsoluteRawSyntax AbsoluteRaw) {
return make(AbsoluteRaw, nullptr);
}
static RC<SyntaxData> make(AbsoluteRawSyntax AbsoluteRaw,
const RC<SyntaxData> &Parent);
/// Returns the raw syntax node for this syntax node.
const RC<RawSyntax> &getRaw() const {
return Raw;
}
const RC<RawSyntax> &getRaw() const { return AbsoluteRaw.getRaw(); }
/// Returns the kind of syntax node this is.
SyntaxKind getKind() const {
return Raw->getKind();
}
SyntaxKind getKind() const { return AbsoluteRaw.getRaw()->getKind(); }
/// Return the parent syntax if there is one.
const SyntaxData * getParent() const {
return Parent;
}
const RC<SyntaxData> &getParent() const { return Parent; }
/// Returns true if this syntax node has a parent.
bool hasParent() const {
@@ -204,20 +162,21 @@ public:
/// Returns the child index of this node in its parent, if it has a parent,
/// otherwise 0.
size_t getIndexInParent() const {
return IndexInParent;
AbsoluteSyntaxPosition::IndexInParentType getIndexInParent() const {
return AbsoluteRaw.getPosition().getIndexInParent();
}
/// Returns the number of children this SyntaxData represents.
size_t getNumChildren() const {
return Raw->getLayout().size();
return AbsoluteRaw.getRaw()->getLayout().size();
}
/// Gets the child at the index specified by the provided cursor,
/// lazily creating it if necessary.
template <typename CursorType>
RC<SyntaxData> getChild(CursorType Cursor) const {
return getChild((size_t)cursorIndex(Cursor));
return getChild(
(AbsoluteSyntaxPosition::IndexInParentType)cursorIndex(Cursor));
}
/// Gets the child at the specified index in this data's children array.
@@ -247,25 +206,22 @@ public:
/// then we ask the AtomicCache in that position to realize its value and
/// cache it. This is safe because AtomicCache only ever mutates its cache
/// one time -- the first initialization that wins a compare_exchange_strong.
RC<SyntaxData> getChild(size_t Index) const {
if (!getRaw()->getChild(Index))
return nullptr;
return getChildren()[Index].getOrCreate([&]() {
return realizeSyntaxNode(Index);
});
}
RC<SyntaxData>
getChild(AbsoluteSyntaxPosition::IndexInParentType Index) const;
/// Calculate the absolute position of this node, use cache if the cache
/// is populated.
AbsolutePosition getAbsolutePosition() const;
/// Get the offset at which the leading trivia of this node starts.
AbsoluteOffsetPosition getAbsolutePositionBeforeLeadingTrivia() const;
/// Calculate the absolute end position of this node, use cache of the immediate
/// next node if populated.
AbsolutePosition getAbsoluteEndPositionAfterTrailingTrivia() const;
/// Get the offset at which the content (excluding trailing trivia) of this
/// node ends.
AbsoluteOffsetPosition getAbsoluteEndPositionBeforeTrailingTrivia() const;
/// Get the absolute position without skipping the leading trivia of this
/// node.
AbsolutePosition getAbsolutePositionBeforeLeadingTrivia() const;
/// Get the offset at which the content of this node (excluding leading
/// trivia) starts.
AbsoluteOffsetPosition getAbsolutePositionAfterLeadingTrivia() const;
/// Get the offset at chiwh the trailing trivia of this node ends.
AbsoluteOffsetPosition getAbsoluteEndPositionAfterTrailingTrivia() const;
/// Returns true if the data node represents type syntax.
bool isType() const;

6
include/swift/Syntax/TokenSyntax.h
View File

@@ -37,7 +37,7 @@ public:
: Syntax(Root, Data) {}
static TokenSyntax missingToken(const tok Kind, OwnedString Text) {
return make<TokenSyntax>(RawSyntax::missing(Kind, Text));
return makeRoot<TokenSyntax>(RawSyntax::missing(Kind, Text));
}
Trivia getLeadingTrivia() const {
@@ -50,12 +50,12 @@ public:
TokenSyntax withLeadingTrivia(const Trivia &Trivia) const {
auto NewRaw = getRaw()->withLeadingTrivia(Trivia.Pieces);
return Data->replaceSelf<TokenSyntax>(NewRaw);
return Data->replacingSelf<TokenSyntax>(NewRaw);
}
TokenSyntax withTrailingTrivia(const Trivia &Trivia) const {
auto NewRaw = getRaw()->withTrailingTrivia(Trivia.Pieces);
return Data->replaceSelf<TokenSyntax>(NewRaw);
return Data->replacingSelf<TokenSyntax>(NewRaw);
}
/* TODO: If we really need them.

4
include/swift/Syntax/Trivia.h.gyb
View File

@@ -99,8 +99,6 @@
namespace swift {
namespace syntax {
class AbsolutePosition;
/// The kind of source trivia, such as spaces, newlines, or comments.
enum class TriviaKind {
% for trivia in TRIVIAS:
@@ -181,8 +179,6 @@ public:
bool isComment() const {
return isCommentTriviaKind(getKind());
}
void accumulateAbsolutePosition(AbsolutePosition &Pos) const;
/// Try to compose this and Next to one TriviaPiece.
/// It returns true if it is succeeded.

25
lib/Parse/Parser.cpp
View File

@@ -320,14 +320,13 @@ std::vector<Token> swift::tokenize(const LangOptions &LangOpts,
return Tokens;
}
std::vector<std::pair<RC<syntax::RawSyntax>, syntax::AbsolutePosition>>
std::vector<std::pair<RC<syntax::RawSyntax>, syntax::AbsoluteOffsetPosition>>
swift::tokenizeWithTrivia(const LangOptions &LangOpts, const SourceManager &SM,
unsigned BufferID, unsigned Offset,
unsigned EndOffset,
DiagnosticEngine *Diags) {
std::vector<std::pair<RC<syntax::RawSyntax>, syntax::AbsolutePosition>>
unsigned EndOffset, DiagnosticEngine *Diags) {
std::vector<std::pair<RC<syntax::RawSyntax>, syntax::AbsoluteOffsetPosition>>
Tokens;
syntax::AbsolutePosition RunningPos;
syntax::AbsoluteOffsetPosition RunningPos(0);
tokenize(
LangOpts, SM, BufferID, Offset, EndOffset, Diags,
@@ -346,13 +345,17 @@ swift::tokenizeWithTrivia(const LangOptions &LangOpts, const SourceManager &SM,
Trivia syntaxTrailingTrivia =
TrailingTrivia.convertToSyntaxTrivia(TrailingTriviaLoc, SM, BufferID);
auto Text = OwnedString::makeRefCounted(Tok.getRawText());
auto ThisToken =
RawSyntax::make(Tok.getKind(), Text, syntaxLeadingTrivia.Pieces,
syntaxTrailingTrivia.Pieces,
SourcePresence::Present);
size_t TextLength = LeadingTrivia.getLength() +
TokRange.getByteLength() +
TrailingTrivia.getLength();
auto ThisToken = RawSyntax::make(
Tok.getKind(), Text, TextLength, syntaxLeadingTrivia.Pieces,
syntaxTrailingTrivia.Pieces, SourcePresence::Present);
auto ThisTokenPos = ThisToken->accumulateAbsolutePosition(RunningPos);
Tokens.push_back({ThisToken, ThisTokenPos.getValue()});
auto ThisTokenPos =
RunningPos.advancedBy(ThisToken->getLeadingTriviaLength());
Tokens.push_back({ThisToken, ThisTokenPos});
RunningPos = RunningPos.advancedBy(ThisToken->getTextLength());
});
return Tokens;

67
lib/Syntax/AbsoluteRawSyntax.cpp Normal file
View File

@@ -0,0 +1,67 @@
//===--- AbsoluteRawSyntax.cpp ----------------------------------*- C++ -*-===//
//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See https://swift.org/LICENSE.txt for license information
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
//
//===----------------------------------------------------------------------===//
#include "swift/Syntax/AbsoluteRawSyntax.h"
using namespace swift;
using namespace swift::syntax;
std::atomic<SyntaxIdentifier::RootIdType> SyntaxIdentifier::NextUnusedRootId(0);
SyntaxIndexInTree
SyntaxIndexInTree::advancedBy(const RC<RawSyntax> &Raw) const {
auto NewIndexInTree = IndexInTree;
if (Raw) {
NewIndexInTree += Raw->getTotalNodes();
}
return SyntaxIndexInTree(NewIndexInTree);
}
SyntaxIndexInTree
SyntaxIndexInTree::reversedBy(const RC<RawSyntax> &Raw) const {
auto NewIndexInTree = IndexInTree;
if (Raw) {
NewIndexInTree -= Raw->getTotalNodes();
}
return SyntaxIndexInTree(NewIndexInTree);
}
SyntaxIndexInTree SyntaxIndexInTree::advancedToFirstChild() const {
auto NewIndexInTree = IndexInTree + 1;
return SyntaxIndexInTree(NewIndexInTree);
}
AbsoluteSyntaxPosition
AbsoluteSyntaxPosition::advancedBy(const RC<RawSyntax> &Raw) const {
OffsetType NewOffset = Offset;
if (Raw) {
NewOffset += Raw->getTextLength();
}
IndexInParentType NewIndexInParent = IndexInParent + 1;
return AbsoluteSyntaxPosition(NewOffset, NewIndexInParent);
}
AbsoluteSyntaxPosition
AbsoluteSyntaxPosition::reversedBy(const RC<RawSyntax> &Raw) const {
OffsetType NewOffset = Offset;
if (Raw) {
NewOffset -= Raw->getTextLength();
}
IndexInParentType NewIndexInParent = IndexInParent - 1;
return AbsoluteSyntaxPosition(NewOffset, NewIndexInParent);
}
raw_ostream &llvm::operator<<(raw_ostream &OS,
swift::syntax::AbsoluteOffsetPosition Pos) {
OS << "Offset " << Pos.getOffset();
return OS;
}

1
lib/Syntax/CMakeLists.txt
View File

@@ -5,6 +5,7 @@ else()
endif()
add_swift_host_library(swiftSyntax STATIC
AbsoluteRawSyntax.cpp
RawSyntax.cpp
Syntax.cpp
SyntaxData.cpp

106
lib/Syntax/RawSyntax.cpp
View File

@@ -67,26 +67,37 @@ void swift::dumpTokenKind(llvm::raw_ostream &OS, tok Kind) {
}
}
// FIXME: If we want thread-safety for tree creation, this needs to be atomic.
unsigned RawSyntax::NextFreeNodeId = 1;
RawSyntax::RawSyntax(SyntaxKind Kind, ArrayRef<RC<RawSyntax>> Layout,
SourcePresence Presence, const RC<SyntaxArena> &Arena,
size_t TextLength, SourcePresence Presence,
const RC<SyntaxArena> &Arena,
llvm::Optional<unsigned> NodeId) {
assert(Kind != SyntaxKind::Token &&
"'token' syntax node must be constructed with dedicated constructor");
RefCount = 0;
size_t TotalSubNodeCount = 0;
for (auto Child : Layout) {
if (Child) {
TotalSubNodeCount += Child->getTotalSubNodeCount() + 1;
}
}
if (NodeId.hasValue()) {
this->NodeId = NodeId.getValue();
NextFreeNodeId = std::max(this->NodeId + 1, NextFreeNodeId);
} else {
this->NodeId = NextFreeNodeId++;
}
Bits.Common.Kind = unsigned(Kind);
Bits.Common.TextLength = TextLength;
Bits.Common.Presence = unsigned(Presence);
Bits.Common.IsToken = false;
Bits.Layout.NumChildren = Layout.size();
Bits.Layout.TextLength = UINT32_MAX;
Bits.Layout.TotalSubNodeCount = TotalSubNodeCount;
Bits.Layout.Kind = unsigned(Kind);
this->Arena = Arena;
@@ -95,7 +106,7 @@ RawSyntax::RawSyntax(SyntaxKind Kind, ArrayRef<RC<RawSyntax>> Layout,
getTrailingObjects<RC<RawSyntax>>());
}
RawSyntax::RawSyntax(tok TokKind, OwnedString Text,
RawSyntax::RawSyntax(tok TokKind, OwnedString Text, size_t TextLength,
ArrayRef<TriviaPiece> LeadingTrivia,
ArrayRef<TriviaPiece> TrailingTrivia,
SourcePresence Presence, const RC<SyntaxArena> &Arena,
@@ -108,8 +119,9 @@ RawSyntax::RawSyntax(tok TokKind, OwnedString Text,
} else {
this->NodeId = NextFreeNodeId++;
}
Bits.Common.Kind = unsigned(SyntaxKind::Token);
Bits.Common.TextLength = TextLength;
Bits.Common.Presence = unsigned(Presence);
Bits.Common.IsToken = true;
Bits.Token.TokenKind = unsigned(TokKind);
Bits.Token.NumLeadingTrivia = LeadingTrivia.size();
Bits.Token.NumTrailingTrivia = TrailingTrivia.size();
@@ -142,7 +154,7 @@ RawSyntax::~RawSyntax() {
}
RC<RawSyntax> RawSyntax::make(SyntaxKind Kind, ArrayRef<RC<RawSyntax>> Layout,
SourcePresence Presence,
size_t TextLength, SourcePresence Presence,
const RC<SyntaxArena> &Arena,
llvm::Optional<unsigned> NodeId) {
auto size = totalSizeToAlloc<RC<RawSyntax>, OwnedString, TriviaPiece>(
@@ -150,10 +162,10 @@ RC<RawSyntax> RawSyntax::make(SyntaxKind Kind, ArrayRef<RC<RawSyntax>> Layout,
void *data = Arena ? Arena->Allocate(size, alignof(RawSyntax))
: ::operator new(size);
return RC<RawSyntax>(
new (data) RawSyntax(Kind, Layout, Presence, Arena, NodeId));
new (data) RawSyntax(Kind, Layout, TextLength, Presence, Arena, NodeId));
}
RC<RawSyntax> RawSyntax::make(tok TokKind, OwnedString Text,
RC<RawSyntax> RawSyntax::make(tok TokKind, OwnedString Text, size_t TextLength,
ArrayRef<TriviaPiece> LeadingTrivia,
ArrayRef<TriviaPiece> TrailingTrivia,
SourcePresence Presence,
@@ -163,9 +175,9 @@ RC<RawSyntax> RawSyntax::make(tok TokKind, OwnedString Text,
0, 1, LeadingTrivia.size() + TrailingTrivia.size());
void *data = Arena ? Arena->Allocate(size, alignof(RawSyntax))
: ::operator new(size);
return RC<RawSyntax>(new (data) RawSyntax(TokKind, Text, LeadingTrivia,
TrailingTrivia, Presence,
Arena, NodeId));
return RC<RawSyntax>(new (data)
RawSyntax(TokKind, Text, TextLength, LeadingTrivia,
TrailingTrivia, Presence, Arena, NodeId));
}
RC<RawSyntax> RawSyntax::append(RC<RawSyntax> NewLayoutElement) const {
@@ -174,11 +186,12 @@ RC<RawSyntax> RawSyntax::append(RC<RawSyntax> NewLayoutElement) const {
NewLayout.reserve(Layout.size() + 1);
std::copy(Layout.begin(), Layout.end(), std::back_inserter(NewLayout));
NewLayout.push_back(NewLayoutElement);
return RawSyntax::make(getKind(), NewLayout, SourcePresence::Present);
return RawSyntax::makeAndCalcLength(getKind(), NewLayout,
SourcePresence::Present);
}
RC<RawSyntax> RawSyntax::replaceChild(CursorIndex Index,
RC<RawSyntax> NewLayoutElement) const {
RC<RawSyntax> RawSyntax::replacingChild(CursorIndex Index,
RC<RawSyntax> NewLayoutElement) const {
auto Layout = getLayout();
std::vector<RC<RawSyntax>> NewLayout;
NewLayout.reserve(Layout.size());
@@ -191,49 +204,7 @@ RC<RawSyntax> RawSyntax::replaceChild(CursorIndex Index,
std::copy(Layout.begin() + Index + 1, Layout.end(),
std::back_inserter(NewLayout));
return RawSyntax::make(getKind(), NewLayout, getPresence());
}
llvm::Optional<AbsolutePosition>
RawSyntax::accumulateAbsolutePosition(AbsolutePosition &Pos) const {
llvm::Optional<AbsolutePosition> Ret;
if (isToken()) {
if (isMissing())
return None;
for (auto &Leader : getLeadingTrivia())
Leader.accumulateAbsolutePosition(Pos);
Ret = Pos;
Pos.addText(getTokenText());
for (auto &Trailer : getTrailingTrivia())
Trailer.accumulateAbsolutePosition(Pos);
} else {
for (auto &Child : getLayout()) {
if (!Child)
continue;
auto Result = Child->accumulateAbsolutePosition(Pos);
if (!Ret && Result)
Ret = Result;
}
}
return Ret;
}
bool RawSyntax::accumulateLeadingTrivia(AbsolutePosition &Pos) const {
if (isToken()) {
if (!isMissing()) {
for (auto &Leader: getLeadingTrivia())
Leader.accumulateAbsolutePosition(Pos);
return true;
}
} else {
for (auto &Child: getLayout()) {
if (!Child || Child->isMissing())
continue;
if (Child->accumulateLeadingTrivia(Pos))
return true;
}
}
return false;
return RawSyntax::makeAndCalcLength(getKind(), NewLayout, getPresence());
}
void RawSyntax::print(llvm::raw_ostream &OS, SyntaxPrintOptions Opts) const {
@@ -313,22 +284,6 @@ void RawSyntax::dump(llvm::raw_ostream &OS, unsigned Indent) const {
OS << ')';
}
void AbsolutePosition::printLineAndColumn(llvm::raw_ostream &OS) const {
OS << getLine() << ':' << getColumn();
}
void AbsolutePosition::dump(llvm::raw_ostream &OS) const {
OS << "(absolute_position ";
OS << "offset=" << getOffset() << " ";
OS << "line=" << getLine() << " ";
OS << "column=" << getColumn();
OS << ')';
}
void AbsolutePosition::dump() const {
dump(llvm::errs());
}
void RawSyntax::Profile(llvm::FoldingSetNodeID &ID, tok TokKind,
OwnedString Text, ArrayRef<TriviaPiece> LeadingTrivia,
ArrayRef<TriviaPiece> TrailingTrivia) {
@@ -351,8 +306,3 @@ void RawSyntax::Profile(llvm::FoldingSetNodeID &ID, tok TokKind,
for (auto &Piece : TrailingTrivia)
Piece.Profile(ID);
}
llvm::raw_ostream &llvm::operator<<(raw_ostream &OS, AbsolutePosition Pos) {
Pos.printLineAndColumn(OS);
return OS;
}

8
lib/Syntax/Syntax.cpp
View File

@@ -17,9 +17,7 @@
using namespace swift;
using namespace swift::syntax;
const RC<RawSyntax> &Syntax::getRaw() const {
return Data->getRaw();
}
const RC<RawSyntax> Syntax::getRaw() const { return Data->getRaw(); }
SyntaxKind Syntax::getKind() const {
return getRaw()->getKind();
@@ -77,9 +75,7 @@ bool Syntax::isMissing() const {
llvm::Optional<Syntax> Syntax::getParent() const {
auto ParentData = getData().getParent();
if (!ParentData) return llvm::None;
return llvm::Optional<Syntax> {
Syntax { Root, ParentData }
};
return llvm::Optional<Syntax>{Syntax{Root, ParentData.get()}};
}
Syntax Syntax::getRoot() const {

10
lib/Syntax/SyntaxBuilders.cpp.gyb
View File

@@ -45,9 +45,9 @@ ${node.name}Builder &
${node.name}Builder::add${child_elt}(${child_elt_type} ${child_elt}) {
auto &raw = Layout[cursorIndex(${node.name}::Cursor::${child.name})];
if (!raw)
raw = RawSyntax::make(SyntaxKind::${child_node.syntax_kind},
{${child_elt}.getRaw()},
SourcePresence::Present, Arena);
raw = RawSyntax::makeAndCalcLength(SyntaxKind::${child_node.syntax_kind},
{${child_elt}.getRaw()},
SourcePresence::Present, Arena);
else
raw = raw->append(${child_elt}.getRaw());
return *this;
@@ -64,9 +64,9 @@ ${node.name}Builder::build() {
% end
% end
% end
auto raw = RawSyntax::make(SyntaxKind::${node.syntax_kind},
auto raw = RawSyntax::makeAndCalcLength(SyntaxKind::${node.syntax_kind},
Layout, SourcePresence::Present, Arena);
return make<${node.name}>(raw);
return makeRoot<${node.name}>(raw);
}
% end

133
lib/Syntax/SyntaxData.cpp
View File

@@ -15,40 +15,26 @@
using namespace swift;
using namespace swift::syntax;
RC<SyntaxData> SyntaxData::make(RC<RawSyntax> Raw,
const SyntaxData *Parent,
CursorIndex IndexInParent) {
auto size = totalSizeToAlloc<AtomicCache<SyntaxData>>(Raw->getNumChildren());
void *data = ::operator new(size);
return RC<SyntaxData>{new (data) SyntaxData(Raw, Parent, IndexInParent)};
RC<SyntaxData> SyntaxData::make(AbsoluteRawSyntax AbsoluteRaw,
const RC<SyntaxData> &Parent) {
// FIXME: Can we use a bump allocator here?
return RC<SyntaxData>{new SyntaxData(AbsoluteRaw, Parent)};
}
bool SyntaxData::isType() const {
return Raw->isType();
}
bool SyntaxData::isType() const { return getRaw()->isType(); }
bool SyntaxData::isStmt() const {
return Raw->isStmt();
}
bool SyntaxData::isStmt() const { return getRaw()->isStmt(); }
bool SyntaxData::isDecl() const {
return Raw->isDecl();
}
bool SyntaxData::isDecl() const { return getRaw()->isDecl(); }
bool SyntaxData::isExpr() const {
return Raw->isExpr();
}
bool SyntaxData::isExpr() const { return getRaw()->isExpr(); }
bool SyntaxData::isPattern() const {
return Raw->isPattern();
}
bool SyntaxData::isPattern() const { return getRaw()->isPattern(); }
bool SyntaxData::isUnknown() const {
return Raw->isUnknown();
}
bool SyntaxData::isUnknown() const { return getRaw()->isUnknown(); }
void SyntaxData::dump(llvm::raw_ostream &OS) const {
Raw->dump(OS, 0);
getRaw()->dump(OS, 0);
OS << '\n';
}
@@ -86,9 +72,7 @@ RC<SyntaxData> SyntaxData::getNextNode() const {
RC<SyntaxData> SyntaxData::getFirstToken() const {
if (getRaw()->isToken()) {
// Get a reference counted version of this
assert(hasParent() && "The syntax tree should not conisist only of the root");
return getParent()->getChild(getIndexInParent());
return getRefCountedThis();
}
for (size_t I = 0, E = getNumChildren(); I < E; ++I) {
@@ -105,32 +89,83 @@ RC<SyntaxData> SyntaxData::getFirstToken() const {
return nullptr;
}
AbsolutePosition SyntaxData::getAbsolutePositionBeforeLeadingTrivia() const {
if (PositionCache.hasValue())
return *PositionCache;
if (auto P = getPreviousNode()) {
auto Result = P->getAbsolutePositionBeforeLeadingTrivia();
P->getRaw()->accumulateAbsolutePosition(Result);
// FIXME: avoid using const_cast.
const_cast<SyntaxData*>(this)->PositionCache = Result;
} else {
const_cast<SyntaxData*>(this)->PositionCache = AbsolutePosition();
RC<SyntaxData> SyntaxData::getLastToken() const {
if (getRaw()->isToken() && !getRaw()->isMissing()) {
return getRefCountedThis();
}
return *PositionCache;
if (getNumChildren() == 0) {
return nullptr;
}
for (int I = getNumChildren() - 1; I >= 0; --I) {
if (auto Child = getChild(I)) {
if (Child->getRaw()->isMissing()) {
continue;
}
if (Child->getRaw()->isToken()) {
return Child;
} else if (auto Token = Child->getLastToken()) {
return Token;
}
}
}
return nullptr;
}
AbsolutePosition SyntaxData::getAbsolutePosition() const {
auto Result = getAbsolutePositionBeforeLeadingTrivia();
getRaw()->accumulateLeadingTrivia(Result);
return Result;
RC<SyntaxData>
SyntaxData::getChild(AbsoluteSyntaxPosition::IndexInParentType Index) const {
if (!getRaw()->getChild(Index)) {
return nullptr;
}
/// FIXME: Start from the back (advancedToEndOfChildren) and reverse from
/// there if Index is closer to the end as a performance improvement?
AbsoluteSyntaxPosition Position =
AbsoluteRaw.getInfo().getPosition().advancedToFirstChild();
SyntaxIdentifier NodeId =
AbsoluteRaw.getInfo().getNodeId().advancedToFirstChild();
for (size_t I = 0; I < Index; ++I) {
Position = Position.advancedBy(getRaw()->getChild(I));
NodeId = NodeId.advancedBy(getRaw()->getChild(I));
}
AbsoluteSyntaxInfo Info(Position, NodeId);
// FIXME: We are leaking here.
const RC<SyntaxData> RefCountedParent = getRefCountedThis();
RC<SyntaxData> Data = SyntaxData::make(
AbsoluteRawSyntax(getRaw()->getChild(Index), Info), RefCountedParent);
auto Data2 = Data;
Data2.resetWithoutRelease();
return Data;
}
AbsolutePosition SyntaxData::getAbsoluteEndPositionAfterTrailingTrivia() const {
if (auto N = getNextNode()) {
return N->getAbsolutePositionBeforeLeadingTrivia();
AbsoluteOffsetPosition
SyntaxData::getAbsolutePositionBeforeLeadingTrivia() const {
return AbsoluteRaw.getPosition();
}
AbsoluteOffsetPosition
SyntaxData::getAbsolutePositionAfterLeadingTrivia() const {
if (auto FirstToken = getFirstToken()) {
return getAbsolutePositionBeforeLeadingTrivia().advancedBy(
FirstToken->getRaw()->getLeadingTriviaLength());
} else {
auto Result = getAbsolutePositionBeforeLeadingTrivia();
getRaw()->accumulateAbsolutePosition(Result);
return Result;
return getAbsolutePositionBeforeLeadingTrivia();
}
}
AbsoluteOffsetPosition
SyntaxData::getAbsoluteEndPositionBeforeTrailingTrivia() const {
if (auto LastToken = getLastToken()) {
return getAbsoluteEndPositionAfterTrailingTrivia().advancedBy(
-LastToken->getRaw()->getTrailingTriviaLength());
} else {
return getAbsoluteEndPositionAfterTrailingTrivia();
}
}
AbsoluteOffsetPosition
SyntaxData::getAbsoluteEndPositionAfterTrailingTrivia() const {
return getAbsolutePositionBeforeLeadingTrivia().advancedBy(
getRaw()->getTextLength());
}

33
lib/Syntax/SyntaxFactory.cpp.gyb
View File

@@ -43,9 +43,8 @@ TokenSyntax SyntaxFactory::makeToken(tok Kind, OwnedString Text,
const Trivia &TrailingTrivia,
SourcePresence Presence,
RC<SyntaxArena> Arena) {
return make<TokenSyntax>(RawSyntax::make(Kind, Text, LeadingTrivia.Pieces,
TrailingTrivia.Pieces, Presence,
Arena));
return makeRoot<TokenSyntax>(RawSyntax::makeAndCalcLength(Kind, Text,
LeadingTrivia.Pieces, TrailingTrivia.Pieces, Presence, Arena));
}
UnknownSyntax
@@ -56,9 +55,9 @@ SyntaxFactory::makeUnknownSyntax(llvm::ArrayRef<TokenSyntax> Tokens,
for (auto &Token : Tokens) {
Layout.push_back(Token.getRaw());
}
auto Raw = RawSyntax::make(SyntaxKind::Unknown, Layout,
SourcePresence::Present, Arena);
return make<UnknownSyntax>(Raw);
auto Raw = RawSyntax::makeAndCalcLength(SyntaxKind::Unknown, Layout,
SourcePresence::Present, Arena);
return makeRoot<UnknownSyntax>(Raw);
}
Syntax SyntaxFactory::makeBlankCollectionSyntax(SyntaxKind Kind) {
@@ -147,13 +146,15 @@ RC<RawSyntax> SyntaxFactory::createRaw(SyntaxKind Kind,
% end
if (I != Elements.size())
return nullptr;
return RawSyntax::make(Kind, Layout, SourcePresence::Present, Arena);
return RawSyntax::makeAndCalcLength(Kind, Layout, SourcePresence::Present,
Arena);
% elif node.is_syntax_collection():
for (auto &E : Elements) {
if (!canServeAsCollectionMemberRaw(SyntaxKind::${node.syntax_kind}, E))
return nullptr;
}
return RawSyntax::make(Kind, Elements, SourcePresence::Present, Arena);
return RawSyntax::makeAndCalcLength(Kind, Elements, SourcePresence::Present,
Arena);
% else:
return nullptr;
% end
@@ -173,7 +174,7 @@ Optional<Syntax> SyntaxFactory::createSyntax(SyntaxKind Kind,
Layout.emplace_back(E.getRaw());
if (auto Raw = createRaw(Kind, Layout, Arena))
return make<Syntax>(Raw);
return makeRoot<Syntax>(Raw);
else
return None;
}
@@ -190,7 +191,7 @@ Optional<Syntax> SyntaxFactory::createSyntax(SyntaxKind Kind,
${node.name}
SyntaxFactory::make${node.syntax_kind}(${child_params},
RC<SyntaxArena> Arena) {
auto Raw = RawSyntax::make(SyntaxKind::${node.syntax_kind}, {
auto Raw = RawSyntax::makeAndCalcLength(SyntaxKind::${node.syntax_kind}, {
% for child in node.children:
% if child.is_optional:
${child.name}.hasValue() ? ${child.name}->getRaw() : nullptr,
@@ -199,7 +200,7 @@ SyntaxFactory::make${node.syntax_kind}(${child_params},
% end
% end
}, SourcePresence::Present, Arena);
return make<${node.name}>(Raw);
return makeRoot<${node.name}>(Raw);
}
% elif node.is_syntax_collection():
${node.name}
@@ -211,9 +212,9 @@ SyntaxFactory::make${node.syntax_kind}(
for (auto &element : elements) {
layout.push_back(element.getRaw());
}
auto raw = RawSyntax::make(SyntaxKind::${node.syntax_kind},
layout, SourcePresence::Present, Arena);
return make<${node.name}>(raw);
auto raw = RawSyntax::makeAndCalcLength(SyntaxKind::${node.syntax_kind},
layout, SourcePresence::Present, Arena);
return makeRoot<${node.name}>(raw);
}
% end
@@ -227,8 +228,8 @@ SyntaxFactory::makeBlank${node.syntax_kind}(RC<SyntaxArena> Arena) {
${make_missing_child(child)},
% end
% end
}, SourcePresence::Present, Arena);
return make<${node.name}>(raw);
}, /*TextLength=*/0, SourcePresence::Present, Arena);
return makeRoot<${node.name}>(raw);
}
% end

8
lib/Syntax/SyntaxNodes.cpp.gyb
View File

@@ -79,9 +79,9 @@ ${node.name} ${node.name}::add${child_elt}(${child_elt_type} ${child_elt}) {
if (raw)
raw = raw->append(${child_elt}.getRaw());
else
raw = RawSyntax::make(SyntaxKind::${child_node.syntax_kind},
{${child_elt}.getRaw()}, SourcePresence::Present);
return Data->replaceChild<${node.name}>(raw, Cursor::${child.name});
raw = RawSyntax::makeAndCalcLength(SyntaxKind::${child_node.syntax_kind},
{${child_elt}.getRaw()}, SourcePresence::Present);
return Data->replacingChild<${node.name}>(raw, Cursor::${child.name});
}
% end
@@ -97,7 +97,7 @@ ${node.name} ${node.name}::with${child.name}(
raw = ${make_missing_child(child)};
% end
}
return Data->replaceChild<${node.name}>(raw, Cursor::${child.name});
return Data->replacingChild<${node.name}>(raw, Cursor::${child.name});
}
% end

16
lib/Syntax/Trivia.cpp.gyb
View File

@@ -73,22 +73,6 @@ bool syntax::isCommentTriviaKind(TriviaKind Kind) {
llvm_unreachable("unknown kind");
}
void TriviaPiece::accumulateAbsolutePosition(AbsolutePosition &Pos) const {
switch (Kind) {
% for trivia in TRIVIAS:
case TriviaKind::${trivia.name}:
% if not trivia.is_collection():
Pos.addText(Text.str());
% elif trivia.is_new_line:
Pos.addNewlines(Count, ${trivia.characters_len()});
% else:
Pos.addColumns(Count);
% end
break;
% end
}
}
bool TriviaPiece::trySquash(const TriviaPiece &Next) {
if (Kind != Next.Kind) { return false; }
switch (Kind) {

16
lib/SyntaxParse/RawSyntaxTokenCache.cpp
View File

@@ -42,16 +42,14 @@ static bool shouldCacheNode(tok TokKind, size_t TextSize,
return true;
}
RC<RawSyntax>
RawSyntaxTokenCache::getToken(RC<SyntaxArena> &Arena, tok TokKind,
OwnedString Text,
ArrayRef<TriviaPiece> LeadingTrivia,
ArrayRef<TriviaPiece> TrailingTrivia) {
RC<RawSyntax> RawSyntaxTokenCache::getToken(
RC<SyntaxArena> &Arena, tok TokKind, size_t TextLength, OwnedString Text,
ArrayRef<TriviaPiece> LeadingTrivia, ArrayRef<TriviaPiece> TrailingTrivia) {
// Determine whether this token is worth to cache.
if (!shouldCacheNode(TokKind, Text.size(), LeadingTrivia, TrailingTrivia)) {
// Do not use cache.
return RawSyntax::make(TokKind, Text, LeadingTrivia, TrailingTrivia,
SourcePresence::Present, Arena);
return RawSyntax::make(TokKind, Text, TextLength, LeadingTrivia,
TrailingTrivia, SourcePresence::Present, Arena);
}
// This node is cacheable. Get or create.
@@ -65,8 +63,8 @@ RawSyntaxTokenCache::getToken(RC<SyntaxArena> &Arena, tok TokKind,
}
// Could not found in the cache. Create it.
auto Raw = RawSyntax::make(TokKind, Text, LeadingTrivia, TrailingTrivia,
SourcePresence::Present, Arena);
auto Raw = RawSyntax::make(TokKind, Text, TextLength, LeadingTrivia,
TrailingTrivia, SourcePresence::Present, Arena);
auto IDRef = ID.Intern(Arena->getAllocator());
auto CacheNode = new (Arena) RawSyntaxCacheNode(Raw, IDRef);
// Keep track of the created RawSyntaxCacheNode so that we can destruct it

2
lib/SyntaxParse/RawSyntaxTokenCache.h
View File

@@ -62,7 +62,7 @@ class RawSyntaxTokenCache {
public:
RC<syntax::RawSyntax> getToken(RC<syntax::SyntaxArena> &Arena, tok TokKind,
OwnedString Text,
size_t TextLength, OwnedString Text,
ArrayRef<syntax::TriviaPiece> LeadingTrivia,
ArrayRef<syntax::TriviaPiece> TrailingTrivia);

11
lib/SyntaxParse/SyntaxTreeCreator.cpp
View File

@@ -97,7 +97,7 @@ Optional<SourceFileSyntax>
SyntaxTreeCreator::realizeSyntaxRoot(OpaqueSyntaxNode rootN,
const SourceFile &SF) {
auto raw = transferOpaqueNode(rootN);
auto rootNode = make<SourceFileSyntax>(raw);
auto rootNode = makeRoot<SourceFileSyntax>(raw);
// Verify the tree if specified.
if (SF.getASTContext().LangOpts.VerifySyntaxTree) {
@@ -132,8 +132,9 @@ SyntaxTreeCreator::recordToken(tok tokenKind,
trailingTriviaLoc, SM, BufferID);
StringRef tokenText = SM.extractText(tokRange, BufferID);
auto ownedText = OwnedString::makeRefCounted(tokenText);
auto raw = TokenCache->getToken(Arena, tokenKind, ownedText,
syntaxLeadingTrivia.Pieces, syntaxTrailingTrivia.Pieces);
auto raw = TokenCache->getToken(Arena, tokenKind, range.getByteLength(),
ownedText, syntaxLeadingTrivia.Pieces,
syntaxTrailingTrivia.Pieces);
OpaqueSyntaxNode opaqueN = raw.get();
raw.resetWithoutRelease();
return opaqueN;
@@ -157,7 +158,9 @@ SyntaxTreeCreator::recordRawSyntax(syntax::SyntaxKind kind,
for (OpaqueSyntaxNode opaqueN : elements) {
parts.push_back(transferOpaqueNode(opaqueN));
}
auto raw = RawSyntax::make(kind, parts, SourcePresence::Present, Arena);
size_t TextLength = range.isValid() ? range.getByteLength() : 0;
auto raw =
RawSyntax::make(kind, parts, TextLength, SourcePresence::Present, Arena);
OpaqueSyntaxNode opaqueN = raw.get();
raw.resetWithoutRelease();
return opaqueN;

56
tools/swift-syntax-test/swift-syntax-test.cpp
View File

@@ -289,26 +289,23 @@ struct ByteBasedSourceRangeSet {
}
};
int getTokensFromFile(unsigned BufferID,
LangOptions &LangOpts,
SourceManager &SourceMgr,
swift::DiagnosticEngine &Diags,
std::vector<std::pair<RC<syntax::RawSyntax>,
syntax::AbsolutePosition>> &Tokens) {
int getTokensFromFile(
unsigned BufferID, LangOptions &LangOpts, SourceManager &SourceMgr,
swift::DiagnosticEngine &Diags,
std::vector<std::pair<RC<syntax::RawSyntax>,
syntax::AbsoluteOffsetPosition>> &Tokens) {
Tokens = tokenizeWithTrivia(LangOpts, SourceMgr, BufferID,
/*Offset=*/0, /*EndOffset=*/0,
&Diags);
return EXIT_SUCCESS;
}
int
getTokensFromFile(const StringRef InputFilename,
LangOptions &LangOpts,
SourceManager &SourceMgr,
DiagnosticEngine &Diags,
std::vector<std::pair<RC<syntax::RawSyntax>,
syntax::AbsolutePosition>> &Tokens) {
int getTokensFromFile(
const StringRef InputFilename, LangOptions &LangOpts,
SourceManager &SourceMgr, DiagnosticEngine &Diags,
unsigned int &OutBufferID,
std::vector<std::pair<RC<syntax::RawSyntax>,
syntax::AbsoluteOffsetPosition>> &Tokens) {
auto Buffer = llvm::MemoryBuffer::getFile(InputFilename);
if (!Buffer) {
Diags.diagnose(SourceLoc(), diag::cannot_open_file,
@@ -316,8 +313,8 @@ getTokensFromFile(const StringRef InputFilename,
return EXIT_FAILURE;
}
auto BufferID = SourceMgr.addNewSourceBuffer(std::move(Buffer.get()));
return getTokensFromFile(BufferID, LangOpts, SourceMgr, Diags, Tokens);
OutBufferID = SourceMgr.addNewSourceBuffer(std::move(Buffer.get()));
return getTokensFromFile(OutBufferID, LangOpts, SourceMgr, Diags, Tokens);
}
void anchorForGetMainExecutable() {}
@@ -674,10 +671,11 @@ int doFullLexRoundTrip(const StringRef InputFilename) {
PrintingDiagnosticConsumer DiagPrinter;
Diags.addConsumer(DiagPrinter);
std::vector<std::pair<RC<syntax::RawSyntax>,
syntax::AbsolutePosition>> Tokens;
if (getTokensFromFile(InputFilename, LangOpts, SourceMgr,
Diags, Tokens) == EXIT_FAILURE) {
unsigned int BufferID;
std::vector<std::pair<RC<syntax::RawSyntax>, syntax::AbsoluteOffsetPosition>>
Tokens;
if (getTokensFromFile(InputFilename, LangOpts, SourceMgr, Diags, BufferID,
Tokens) == EXIT_FAILURE) {
return EXIT_FAILURE;
}
@@ -695,15 +693,20 @@ int doDumpRawTokenSyntax(const StringRef InputFile) {
PrintingDiagnosticConsumer DiagPrinter;
Diags.addConsumer(DiagPrinter);
std::vector<std::pair<RC<syntax::RawSyntax>,
syntax::AbsolutePosition>> Tokens;
if (getTokensFromFile(InputFile, LangOpts, SourceMgr, Diags, Tokens) ==
EXIT_FAILURE) {
unsigned int BufferID;
std::vector<std::pair<RC<syntax::RawSyntax>, syntax::AbsoluteOffsetPosition>>
Tokens;
if (getTokensFromFile(InputFile, LangOpts, SourceMgr, Diags, BufferID,
Tokens) == EXIT_FAILURE) {
return EXIT_FAILURE;
}
for (auto TokAndPos : Tokens) {
llvm::outs() << TokAndPos.second << "\n";
SourceLoc Loc =
SourceMgr.getLocForOffset(BufferID, TokAndPos.second.getOffset());
unsigned Line, Column;
std::tie(Line, Column) = SourceMgr.getPresumedLineAndColumnForLoc(Loc);
llvm::outs() << Line << ":" << Column << "\n";
TokAndPos.first->dump(llvm::outs());
llvm::outs() << "\n";
}
@@ -824,8 +827,7 @@ int dumpEOFSourceLoc(const char *MainExecutablePath,
// To ensure the correctness of position when translated to line & column
// pair.
if (SourceMgr.getPresumedLineAndColumnForLoc(EndLoc) !=
AbPos.getLineAndColumn()) {
if (SourceMgr.getLocOffsetInBuffer(EndLoc, BufferId) != AbPos.getOffset()) {
llvm::outs() << "locations should be identical";
return EXIT_FAILURE;
}

37
unittests/Syntax/AbsolutePositionTests.cpp
View File

@@ -1,37 +0,0 @@
#include "swift/Syntax/SyntaxFactory.h"
#include "swift/Syntax/SyntaxNodes.h"
#include "swift/Syntax/SyntaxBuilders.h"
#include "llvm/ADT/SmallString.h"
#include "gtest/gtest.h"
using namespace swift;
using namespace swift::syntax;
TEST(PositionTests, AbsolutePosition1) {
Trivia Leading;
Leading.Pieces = {TriviaPiece::newlines(2), TriviaPiece::carriageReturns(2),
TriviaPiece::carriageReturnLineFeeds(2)};
auto Token = SyntaxFactory::makeIdentifier("aaa", Leading, {});
AbsolutePosition Pos = Token.getAbsolutePosition();
ASSERT_EQ(7u, Pos.getLine());
ASSERT_EQ(1u, Pos.getColumn());
ASSERT_EQ(8u, Pos.getOffset());
AbsolutePosition EndPos = Token.getAbsoluteEndPositionAfterTrailingTrivia();
ASSERT_EQ(7u, EndPos.getLine());
ASSERT_EQ(4u, EndPos.getColumn());
ASSERT_EQ(11u, EndPos.getOffset());
}
TEST(PositionTests, AbsolutePosition2) {
Trivia Leading;
Leading.Pieces = { TriviaPiece::blockComment("/* \n\r\r\n */") };
auto Token = SyntaxFactory::makeIdentifier("aaa", Leading, {});
AbsolutePosition Pos = Token.getAbsolutePosition();
ASSERT_EQ(4u, Pos.getLine());
ASSERT_EQ(4u, Pos.getColumn());
ASSERT_EQ(10u, Pos.getOffset());
AbsolutePosition EndPos = Token.getAbsoluteEndPositionAfterTrailingTrivia();
ASSERT_EQ(4u, EndPos.getLine());
ASSERT_EQ(7u, EndPos.getColumn());
ASSERT_EQ(13u, EndPos.getOffset());
}

2
unittests/Syntax/CMakeLists.txt
View File

@@ -2,14 +2,12 @@ add_swift_unittest(SwiftSyntaxTests
DeclSyntaxTests.cpp
ExprSyntaxTests.cpp
GenericSyntaxTests.cpp
RawSyntaxTests.cpp
StmtSyntaxTests.cpp
SyntaxCollectionTests.cpp
ThreadSafeCachingTests.cpp
TriviaTests.cpp
TypeSyntaxTests.cpp
UnknownSyntaxTests.cpp
AbsolutePositionTests.cpp
)
target_link_libraries(SwiftSyntaxTests

40
unittests/Syntax/RawSyntaxTests.cpp
View File

@@ -1,40 +0,0 @@
#include "swift/Parse/Token.h"
#include "swift/Syntax/RawSyntax.h"
#include "swift/Syntax/SyntaxFactory.h"
#include "llvm/ADT/SmallString.h"
#include "gtest/gtest.h"
using namespace swift;
using namespace swift::syntax;
// TODO
TEST(RawSyntaxTests, accumulateAbsolutePosition1) {
auto Token = RawSyntax::make(tok::identifier,
OwnedString("aaa"),
{
TriviaPiece::newlines(2),
TriviaPiece::carriageReturns(2),
TriviaPiece::carriageReturnLineFeeds(2)
},
{ },
SourcePresence::Present);
AbsolutePosition Pos;
Token->accumulateAbsolutePosition(Pos);
ASSERT_EQ(7u, Pos.getLine());
ASSERT_EQ(4u, Pos.getColumn());
ASSERT_EQ(11u, Pos.getOffset());
}
TEST(RawSyntaxTests, accumulateAbsolutePosition2) {
auto Token = RawSyntax::make(tok::identifier,
OwnedString("aaa"),
{TriviaPiece::blockComment("/* \n\r\r\n */")},
{ },
SourcePresence::Present);
AbsolutePosition Pos;
Token->accumulateAbsolutePosition(Pos);
ASSERT_EQ(4u, Pos.getLine());
ASSERT_EQ(7u, Pos.getColumn());
ASSERT_EQ(13u, Pos.getOffset());
}