[libSyntax] Restructure RawSyntax to more closely resemble the SwiftSyntax implementation

include/swift/Parse/Parser.h

@@ -59,7 +59,6 @@ namespace swift {
   struct EnumElementInfo;
   
   namespace syntax {
-    class AbsolutePosition;
     class RawSyntax;
     enum class SyntaxKind;
   }// end of syntax namespace
@@ -1817,14 +1816,10 @@ bool isKeywordPossibleDeclStart(const Token &Tok);
 
 /// Lex and return a vector of `TokenSyntax` tokens, which include
 /// leading and trailing trivia.
-std::vector<std::pair<RC<syntax::RawSyntax>,
+std::vector<std::pair<RC<syntax::RawSyntax>, syntax::AbsoluteOffsetPosition>>
-                                 syntax::AbsolutePosition>>
+tokenizeWithTrivia(const LangOptions &LangOpts, const SourceManager &SM,
-tokenizeWithTrivia(const LangOptions &LangOpts,
+                   unsigned BufferID, unsigned Offset = 0,
-                   const SourceManager &SM,
+                   unsigned EndOffset = 0, DiagnosticEngine *Diags = nullptr);
-                   unsigned BufferID,
-                   unsigned Offset = 0,
-                   unsigned EndOffset = 0,
-                   DiagnosticEngine *Diags = nullptr);
 } // end namespace swift
 
 #endif

include/swift/Parse/SyntaxParsingCache.h

@@ -48,8 +48,8 @@ struct SourceEdit {
 };
 
 struct SyntaxReuseRegion {
-  AbsolutePosition Start;
+  AbsoluteOffsetPosition Start;
-  AbsolutePosition End;
+  AbsoluteOffsetPosition End;
 };
 
 class SyntaxParsingCache {

include/swift/Syntax/AbsoluteRawSyntax.h

@@ -0,0 +1,244 @@
+//===--- AbsoluteRawSyntax.h ------------------------------------*- C++ -*-===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SWIFT_SYNTAX_ABSOLUTERAWSYNTAX_H
+#define SWIFT_SYNTAX_ABSOLUTERAWSYNTAX_H
+
+#include "swift/Basic/SourceLoc.h"
+#include "swift/Syntax/RawSyntax.h"
+
+namespace swift {
+namespace syntax {
+
+/// Type-safe wrapper around a \c size_t that describes a node's index in its
+/// tree. This represents the index at which the node will be encountered during
+/// a depth-first traversal of the tree.
+class SyntaxIndexInTree {
+  size_t IndexInTree;
+
+  explicit SyntaxIndexInTree(size_t IndexInTree) : IndexInTree(IndexInTree) {}
+
+public:
+  static SyntaxIndexInTree zero() { return SyntaxIndexInTree(0); }
+
+  /// Assuming that this index points to the start of \p Raw, advance it so that
+  /// it points to the next sibling of \p Raw.
+  SyntaxIndexInTree advancedBy(const RC<RawSyntax> &Raw) const;
+
+  /// Assuming that this index points to the next sibling of \p Raw, reverse it
+  /// so that it points to the start of \p Raw.
+  SyntaxIndexInTree reversedBy(const RC<RawSyntax> &Raw) const;
+
+  /// Advance this index to point to its first immediate child.
+  SyntaxIndexInTree advancedToFirstChild() const;
+};
+
+/// A syntax identifier that globally identifies a \c Syntax node.
+/// In contrast to \c NodeId of \c RawSyntax, this also takes into account the
+/// node's position in the tree. For example, a source file may contain two
+/// int literals with value 0. The corresponding \c RawSyntax nodes can share
+/// the same \c NodeId. However, because the literals occur in different
+/// locations of the syntax tree, their \c SyntaxIdentifiers are different.
+class SyntaxIdentifier {
+public:
+  using RootIdType = size_t;
+
+private:
+  static std::atomic<RootIdType> NextUnusedRootId;
+
+  /// An integer that identifies the tree in which the node represented by this
+  /// identifier lives.
+  RootIdType RootId;
+
+  /// The position of the node represented by this identifier in the syntax
+  /// tree.
+  SyntaxIndexInTree IndexInTree;
+
+public:
+  SyntaxIdentifier(RootIdType RootId, SyntaxIndexInTree IndexInTree)
+      : RootId(RootId), IndexInTree(IndexInTree) {
+    assert(RootId < NextUnusedRootId && "New RootIds should only be created "
+                                        "using the newRoot constructor.");
+  }
+
+  /// Create a SyntaxIdentifier that refers to the root of a new syntax tree.
+  static SyntaxIdentifier newRoot() {
+    return SyntaxIdentifier(NextUnusedRootId++, SyntaxIndexInTree::zero());
+  }
+
+  RootIdType getRootId() const { return RootId; }
+  SyntaxIndexInTree getIndexInTree() const { return IndexInTree; }
+
+  /// Assuming that this identifier points to the start of \p Raw, advance it so
+  /// that it points to the next sibling of \p Raw.
+  SyntaxIdentifier advancedBy(const RC<RawSyntax> &Raw) const {
+    auto NewIndexInTree = IndexInTree.advancedBy(Raw);
+    return SyntaxIdentifier(RootId, NewIndexInTree);
+  }
+
+  /// Assuming that this identifier points to the next sibling of \p Raw,
+  /// reverse it so that it points to the start of \p Raw.
+  SyntaxIdentifier reversedBy(const RC<RawSyntax> &Raw) const {
+    auto NewIndexInTree = IndexInTree.reversedBy(Raw);
+    return SyntaxIdentifier(RootId, NewIndexInTree);
+  }
+
+  /// Get the identifier of the first immediate child.
+  SyntaxIdentifier advancedToFirstChild() const {
+    auto NewIndexInTree = IndexInTree.advancedToFirstChild();
+    return SyntaxIdentifier(RootId, NewIndexInTree);
+  }
+};
+
+/// Represents a node's position in a syntax tree, described by its overal
+/// textual offset and the position within its parent.
+class AbsoluteSyntaxPosition {
+public:
+  using OffsetType = uint32_t;
+  using IndexInParentType = uint32_t;
+
+private:
+  /// The text offset where this node starts within its syntax tree, counted in
+  /// UTF-8 bytes.
+  OffsetType Offset;
+
+  /// The node's index within its parent, i.e.
+  /// `node.parent.childAt(IndexInParent) = node`.
+  IndexInParentType IndexInParent;
+
+public:
+  AbsoluteSyntaxPosition(OffsetType Offset, IndexInParentType IndexInParent)
+      : Offset(Offset), IndexInParent(IndexInParent) {}
+
+  /// Create a new \c AbsoluteSyntaxPosition that refers to the root of a syntax
+  /// tree.
+  static AbsoluteSyntaxPosition forRoot() {
+    return AbsoluteSyntaxPosition(0, 0);
+  }
+
+  OffsetType getOffset() const { return Offset; }
+  IndexInParentType getIndexInParent() const { return IndexInParent; }
+
+  /// Assuming that this position points to the start of \p Raw, advance it so
+  /// that it points to the next sibling of \p Raw.
+  AbsoluteSyntaxPosition advancedBy(const RC<RawSyntax> &Raw) const;
+
+  /// Assuming that this position points to the next sibling of \p Raw, reverse
+  /// it so that it points to the start of \p Raw.
+  AbsoluteSyntaxPosition reversedBy(const RC<RawSyntax> &Raw) const;
+
+  /// Get the position of the node's first immediate child.
+  AbsoluteSyntaxPosition advancedToFirstChild() const {
+    return AbsoluteSyntaxPosition(Offset, 0);
+  }
+};
+
+/// A type-safe wrapper that describes a node's textual position within a source
+/// file, represented by its UTF-8 byte offset from the start.
+class AbsoluteOffsetPosition {
+  AbsoluteSyntaxPosition::OffsetType Offset;
+
+public:
+  explicit AbsoluteOffsetPosition(AbsoluteSyntaxPosition::OffsetType Offset)
+      : Offset(Offset) {}
+  AbsoluteOffsetPosition(AbsoluteSyntaxPosition Position)
+      : Offset(Position.getOffset()) {}
+
+  AbsoluteSyntaxPosition::OffsetType getOffset() const { return Offset; }
+
+  /// Return a position that has been advanced by \p Advance UTF-8 bytes.s
+  AbsoluteOffsetPosition advancedBy(int Advance) {
+    return AbsoluteOffsetPosition(Offset + Advance);
+  }
+};
+
+/// Various information that enrich a \c RawSyntax node with information on how
+/// it's located within the syntax tree.
+class AbsoluteSyntaxInfo {
+  AbsoluteSyntaxPosition Position;
+  SyntaxIdentifier NodeId;
+
+public:
+  AbsoluteSyntaxInfo(AbsoluteSyntaxPosition Position, SyntaxIdentifier NodeId)
+      : Position(Position), NodeId(NodeId) {}
+
+  static AbsoluteSyntaxInfo forRoot() {
+    return AbsoluteSyntaxInfo(AbsoluteSyntaxPosition::forRoot(),
+                              SyntaxIdentifier::newRoot());
+  }
+
+  AbsoluteSyntaxPosition getPosition() const { return Position; }
+  SyntaxIdentifier getNodeId() const { return NodeId; }
+
+  /// Assuming that this info points to the start of \p Raw, advance it so
+  /// that it points to the next sibling of \p Raw.
+  AbsoluteSyntaxInfo advancedBy(const RC<RawSyntax> &Raw) const {
+    auto NewNodeId = NodeId.advancedBy(Raw);
+    auto NewPosition = Position.advancedBy(Raw);
+    return AbsoluteSyntaxInfo(NewPosition, NewNodeId);
+  }
+
+  /// Assuming that this info points to the next sibling of \p Raw, reverse
+  /// it so that it points to the start of \p Raw.
+  AbsoluteSyntaxInfo reversedBy(const RC<RawSyntax> &Raw) const {
+    auto NewNodeId = NodeId.reversedBy(Raw);
+    auto NewPosition = Position.reversedBy(Raw);
+    return AbsoluteSyntaxInfo(NewPosition, NewNodeId);
+  }
+
+  /// Get the information of the node's first immediate child.
+  AbsoluteSyntaxInfo advancedToFirstChild() const {
+    auto NewNodeId = NodeId.advancedToFirstChild();
+    auto NewPosition = Position.advancedToFirstChild();
+    return AbsoluteSyntaxInfo(NewPosition, NewNodeId);
+  }
+};
+
+/// A \c RawSyntax node that is enrichted with information of its position
+/// within the syntax tree it lives in.
+struct AbsoluteRawSyntax {
+  const RC<RawSyntax> Raw;
+  const AbsoluteSyntaxInfo Info;
+
+public:
+  AbsoluteRawSyntax(const RC<RawSyntax> &Raw, AbsoluteSyntaxInfo Info)
+      : Raw(Raw), Info(Info) {}
+
+  /// Construct a \c AbsoluteRawSyntax for a \c RawSyntax node that represents
+  /// the syntax tree's root.
+  static AbsoluteRawSyntax forRoot(const RC<RawSyntax> &Raw) {
+    return AbsoluteRawSyntax(Raw, AbsoluteSyntaxInfo::forRoot());
+  }
+
+  const RC<RawSyntax> &getRaw() const { return Raw; }
+
+  AbsoluteSyntaxInfo getInfo() const { return Info; }
+
+  /// Get the position at which the leading triva of this node starts.
+  AbsoluteSyntaxPosition getPosition() const { return Info.getPosition(); };
+
+  SyntaxIdentifier getNodeId() const { return Info.getNodeId(); };
+
+  AbsoluteSyntaxPosition::IndexInParentType getIndexInParent() const {
+    return getPosition().getIndexInParent();
+  }
+};
+
+} // end namespace syntax
+} // end namespace swift
+
+namespace llvm {
+raw_ostream &operator<<(raw_ostream &OS,
+                        swift::syntax::AbsoluteOffsetPosition Pos);
+} // end namespace llvm
+
+#endif // SWIFT_SYNTAX_ABSOLUTERAWSYNTAX_H

include/swift/Syntax/RawSyntax.h

@@ -126,78 +126,6 @@ template <typename CursorType> constexpr CursorIndex cursorIndex(CursorType C) {
   return static_cast<CursorIndex>(C);
 }
 
-/// An absolute position in a source file as text - the absolute offset from
-/// the start, line, and column.
-class AbsolutePosition {
-  uintptr_t Offset = 0;
-  uint32_t Line = 1;
-  uint32_t Column = 1;
-
-public:
-  /// Add some number of columns to the position.
-  void addColumns(uint32_t Columns) {
-    Column += Columns;
-    Offset += Columns;
-  }
-
-  /// Add some number of newlines to the position, resetting the column.
-  /// Size is byte size of newline char.
-  /// '\n' and '\r' are 1, '\r\n' is 2.
-  void addNewlines(uint32_t NewLines, uint32_t Size) {
-    Line += NewLines;
-    Column = 1;
-    Offset += NewLines * Size;
-  }
-  
-  /// Use some text as a reference for adding to the absolute position,
-  /// taking note of newlines, etc.
-  void addText(StringRef Str) {
-    const char * C = Str.begin();
-    while (C != Str.end()) {
-      switch (*C++) {
-      case '\n':
-        addNewlines(1, 1);
-        break;
-      case '\r':
-        if (C != Str.end() && *C == '\n') {
-          addNewlines(1, 2);
-          ++C;
-        } else {
-          addNewlines(1, 1);
-        }
-        break;
-      default:
-        addColumns(1);
-        break;
-      }
-    }
-  }
-
-  /// Get the line number of this position.
-  uint32_t getLine() const { return Line; }
-
-  /// Get the column number of this position.
-  uint32_t getColumn() const { return Column; }
-
-  /// Get the line and column number of this position.
-  std::pair<uint32_t, uint32_t> getLineAndColumn() const {
-    return {Line, Column};
-  }
-
-  /// Get the absolute offset of this position, suitable for indexing into a
-  /// buffer if applicable.
-  uintptr_t getOffset() const { return Offset; }
-
-  /// Print the line and column as "l:c" to the given output stream.
-  void printLineAndColumn(llvm::raw_ostream &OS) const;
-
-  /// Dump a description of this position to the given output stream
-  /// for debugging.
-  void dump(llvm::raw_ostream &OS) const;
-
-  SWIFT_DEBUG_DUMP;
-};
-
 /// An indicator of whether a Syntax node was found or written in the source.
 ///
 /// This is not an 'implicit' bit.
@@ -239,33 +167,35 @@ class RawSyntax final
   RC<SyntaxArena> Arena;
 
   union {
-    uint64_t OpaqueBits;
     struct {
-      /// The kind of syntax this node represents.
+      // FIXME: Reduce TextLength to 30 bits so that common fits in 4 bytes?
-      unsigned Kind : bitmax(NumSyntaxKindBits, 8);
+      /// Number of bytes this node takes up spelled out in the source code
+      unsigned TextLength : 32;
       /// Whether this piece of syntax was actually present in the source.
       unsigned Presence : 1;
+      unsigned IsToken : 1;
     } Common;
-    enum { NumRawSyntaxBits = bitmax(NumSyntaxKindBits, 8) + 1 };
+    enum { NumRawSyntaxBits = 32 + 1 + 1 };
 
     // For "layout" nodes.
     struct {
-      static_assert(NumRawSyntaxBits <= 32,
+      static_assert(NumRawSyntaxBits <= 64,
-                    "Only 32 bits reserved for standard syntax bits");
+                    "Only 64 bits reserved for standard syntax bits");
-      uint64_t : bitmax(NumRawSyntaxBits, 32); // align to 32 bits
+      uint64_t : bitmax(NumRawSyntaxBits, 64); // align to 32 bits
       /// Number of children this "layout" node has.
       unsigned NumChildren : 32;
-      /// Number of bytes this node takes up spelled out in the source code
+      /// Total number of sub nodes, i.e. number of transitive children of this
-      /// A value of UINT32_MAX indicates that the text length has not been
+      /// node. This does not include the node itself.
-      /// computed yet.
+      unsigned TotalSubNodeCount : 32;
-      unsigned TextLength : 32;
+      /// The kind of syntax this node represents.
+      unsigned Kind : bitmax(NumSyntaxKindBits, 8);
     } Layout;
 
     // For "token" nodes.
     struct {
-      static_assert(NumRawSyntaxBits <= 16,
+      static_assert(NumRawSyntaxBits <= 64,
-                    "Only 16 bits reserved for standard syntax bits");
+                    "Only 64 bits reserved for standard syntax bits");
-      uint64_t : bitmax(NumRawSyntaxBits, 16); // align to 16 bits
+      uint64_t : bitmax(NumRawSyntaxBits, 64); // align to 16 bits
       /// The kind of token this "token" node represents.
       unsigned TokenKind : 16;
       /// Number of leading  trivia pieces.
@@ -293,7 +223,7 @@ class RawSyntax final
   /// underlying storage.
   /// If \p NodeId is \c None, the next free NodeId is used, if it is passed,
   /// the caller needs to assure that the node ID has not been used yet.
-  RawSyntax(SyntaxKind Kind, ArrayRef<RC<RawSyntax>> Layout,
+  RawSyntax(SyntaxKind Kind, ArrayRef<RC<RawSyntax>> Layout, size_t TextLength,
             SourcePresence Presence, const RC<SyntaxArena> &Arena,
             llvm::Optional<SyntaxNodeId> NodeId);
   /// Constructor for creating token nodes
@@ -301,7 +231,8 @@ class RawSyntax final
   /// underlying storage.
   /// If \p NodeId is \c None, the next free NodeId is used, if it is passed,
   /// the caller needs to assure that the NodeId has not been used yet.
-  RawSyntax(tok TokKind, OwnedString Text, ArrayRef<TriviaPiece> LeadingTrivia,
+  RawSyntax(tok TokKind, OwnedString Text, size_t TextLength,
+            ArrayRef<TriviaPiece> LeadingTrivia,
             ArrayRef<TriviaPiece> TrailingTrivia, SourcePresence Presence,
             const RC<SyntaxArena> &Arena, llvm::Optional<SyntaxNodeId> NodeId);
 
@@ -348,47 +279,93 @@ public:
 
   /// Make a raw "layout" syntax node.
   static RC<RawSyntax> make(SyntaxKind Kind, ArrayRef<RC<RawSyntax>> Layout,
-                            SourcePresence Presence,
+                            size_t TextLength, SourcePresence Presence,
+                            const RC<SyntaxArena> &Arena,
+                            llvm::Optional<SyntaxNodeId> NodeId = llvm::None);
+
+  static RC<RawSyntax> make(SyntaxKind Kind, ArrayRef<RC<RawSyntax>> Layout,
+                            size_t TextLength, SourcePresence Presence,
                             llvm::Optional<SyntaxNodeId> NodeId = llvm::None) {
-    RC<SyntaxArena> Arena = nullptr;
+    return make(Kind, Layout, TextLength, Presence, /*Arena=*/nullptr, NodeId);
-    return make(Kind, Layout, Presence, Arena, NodeId);
   }
 
-  /// Make a raw "layout" syntax node that was allocated in \p Arena.
+  static RC<RawSyntax>
-  static RC<RawSyntax> make(SyntaxKind Kind, ArrayRef<RC<RawSyntax>> Layout,
+  makeAndCalcLength(SyntaxKind Kind, ArrayRef<RC<RawSyntax>> Layout,
+                    SourcePresence Presence, const RC<SyntaxArena> &Arena,
+                    llvm::Optional<SyntaxNodeId> NodeId = llvm::None) {
+    size_t TextLength = 0;
+    for (auto Child : Layout) {
+      if (Child) {
+        TextLength += Child->getTextLength();
+      }
+    }
+    return make(Kind, Layout, TextLength, Presence, Arena, NodeId);
+  }
+
+  static RC<RawSyntax>
+  makeAndCalcLength(SyntaxKind Kind, ArrayRef<RC<RawSyntax>> Layout,
+                    SourcePresence Presence,
+                    llvm::Optional<SyntaxNodeId> NodeId = llvm::None) {
+    return makeAndCalcLength(Kind, Layout, Presence, /*Arena=*/nullptr, NodeId);
+  }
+
+  /// Make a raw "token" syntax node.
+  static RC<RawSyntax> make(tok TokKind, OwnedString Text, size_t TextLength,
+                            ArrayRef<TriviaPiece> LeadingTrivia,
+                            ArrayRef<TriviaPiece> TrailingTrivia,
                             SourcePresence Presence,
                             const RC<SyntaxArena> &Arena,
                             llvm::Optional<SyntaxNodeId> NodeId = llvm::None);
 
-  /// Make a raw "token" syntax node.
+  static RC<RawSyntax> make(tok TokKind, OwnedString Text, size_t TextLength,
-  static RC<RawSyntax> make(tok TokKind, OwnedString Text,
                             ArrayRef<TriviaPiece> LeadingTrivia,
                             ArrayRef<TriviaPiece> TrailingTrivia,
                             SourcePresence Presence,
                             llvm::Optional<SyntaxNodeId> NodeId = llvm::None) {
-    RC<SyntaxArena> Arena = nullptr;
+    return make(TokKind, Text, TextLength, LeadingTrivia, TrailingTrivia,
-    return make(TokKind, Text, LeadingTrivia, TrailingTrivia, Presence, Arena,
+                Presence, /*Arena=*/nullptr, NodeId);
-                NodeId);
   }
 
   /// Make a raw "token" syntax node that was allocated in \p Arena.
-  static RC<RawSyntax> make(tok TokKind, OwnedString Text,
+  static RC<RawSyntax>
-                            ArrayRef<TriviaPiece> LeadingTrivia,
+  makeAndCalcLength(tok TokKind, OwnedString Text,
-                            ArrayRef<TriviaPiece> TrailingTrivia,
+                    ArrayRef<TriviaPiece> LeadingTrivia,
-                            SourcePresence Presence,
+                    ArrayRef<TriviaPiece> TrailingTrivia,
-                            const RC<SyntaxArena> &Arena,
+                    SourcePresence Presence, const RC<SyntaxArena> &Arena,
-                            llvm::Optional<SyntaxNodeId> NodeId = llvm::None);
+                    llvm::Optional<SyntaxNodeId> NodeId = llvm::None) {
+    size_t TextLength = 0;
+    if (Presence != SourcePresence::Missing) {
+      for (auto Trivia : LeadingTrivia) {
+        TextLength += Trivia.getTextLength();
+      }
+      TextLength += Text.size();
+      for (auto Trivia : TrailingTrivia) {
+        TextLength += Trivia.getTextLength();
+      }
+    }
+    return make(TokKind, Text, TextLength, LeadingTrivia, TrailingTrivia,
+                Presence, Arena, NodeId);
+  }
+
+  static RC<RawSyntax> makeAndCalcLength(
+      tok TokKind, OwnedString Text, ArrayRef<TriviaPiece> LeadingTrivia,
+      ArrayRef<TriviaPiece> TrailingTrivia, SourcePresence Presence,
+      llvm::Optional<SyntaxNodeId> NodeId = llvm::None) {
+    return makeAndCalcLength(TokKind, Text, LeadingTrivia, TrailingTrivia,
+                             Presence, /*Arena=*/nullptr, NodeId);
+  }
 
   /// Make a missing raw "layout" syntax node.
   static RC<RawSyntax> missing(SyntaxKind Kind,
                                RC<SyntaxArena> Arena = nullptr) {
-    return make(Kind, {}, SourcePresence::Missing, Arena);
+    return make(Kind, {}, /*TextLength=*/0, SourcePresence::Missing, Arena);
   }
 
   /// Make a missing raw "token" syntax node.
   static RC<RawSyntax> missing(tok TokKind, OwnedString Text,
                                RC<SyntaxArena> Arena = nullptr) {
-    return make(TokKind, Text, {}, {}, SourcePresence::Missing, Arena);
+    return make(TokKind, Text, /*TextLength=*/0, {}, {},
+                SourcePresence::Missing, Arena);
   }
 
   /// @}
@@ -398,7 +375,25 @@ public:
   }
 
   SyntaxKind getKind() const {
-    return static_cast<SyntaxKind>(Bits.Common.Kind);
+    if (Bits.Common.IsToken) {
+      return SyntaxKind::Token;
+    } else {
+      return static_cast<SyntaxKind>(Bits.Layout.Kind);
+    }
+  }
+
+  /// Get the number of nodes included in the subtree spanned by this node.
+  /// This includes all transitive children and this node itself.
+  size_t getTotalNodes() { return getTotalSubNodeCount() + 1; }
+
+  /// Get the number of transitive children of this node. This does not include
+  /// the node itself.
+  size_t getTotalSubNodeCount() {
+    if (isToken()) {
+      return 0;
+    } else {
+      return Bits.Layout.TotalSubNodeCount;
+    }
   }
 
   /// Get an ID for this node that is stable across incremental parses
@@ -430,7 +425,7 @@ public:
   bool isUnknown() const { return isUnknownKind(getKind()); }
 
   /// Return true if this raw syntax node is a token.
-  bool isToken() const { return isTokenKind(getKind()); }
+  bool isToken() const { return Bits.Common.IsToken; }
 
   /// \name Getter routines for SyntaxKind::Token.
   /// @{
@@ -476,8 +471,9 @@ public:
   /// trivia instead.
   RC<RawSyntax>
   withLeadingTrivia(ArrayRef<TriviaPiece> NewLeadingTrivia) const {
-    return make(getTokenKind(), getOwnedTokenText(), NewLeadingTrivia,
+    return makeAndCalcLength(getTokenKind(), getOwnedTokenText(),
-                getTrailingTrivia(), getPresence());
+                             NewLeadingTrivia, getTrailingTrivia(),
+                             getPresence());
   }
 
   RC<RawSyntax> withLeadingTrivia(Trivia NewLeadingTrivia) const {
@@ -488,8 +484,9 @@ public:
   /// trivia instead.
   RC<RawSyntax>
   withTrailingTrivia(ArrayRef<TriviaPiece> NewTrailingTrivia) const {
-    return make(getTokenKind(), getOwnedTokenText(), getLeadingTrivia(),
+    return makeAndCalcLength(getTokenKind(), getOwnedTokenText(),
-                NewTrailingTrivia, getPresence());
+                             getLeadingTrivia(), NewTrailingTrivia,
+                             getPresence());
   }
 
   RC<RawSyntax> withTrailingTrivia(Trivia NewTrailingTrivia) const {
@@ -521,23 +518,7 @@ public:
   }
 
   /// Return the number of bytes this node takes when spelled out in the source
-  size_t getTextLength() {
+  size_t getTextLength() { return Bits.Common.TextLength; }
-    // For tokens the computation of the length is fast enough to justify the
-    // space for caching it. For layout nodes, we cache the length to avoid
-    // traversing the tree
-
-    // FIXME: Or would it be sensible to cache the size of token nodes as well?
-    if (isToken()) {
-      AbsolutePosition Pos;
-      accumulateAbsolutePosition(Pos);
-      return Pos.getOffset();
-    } else {
-      if (Bits.Layout.TextLength == UINT32_MAX) {
-        Bits.Layout.TextLength = computeTextLength();
-      }
-      return Bits.Layout.TextLength;
-    }
-  }
 
   /// @}
 
@@ -550,22 +531,26 @@ public:
 
   /// Return a new raw syntax node with the given new layout element replacing
   /// another at some cursor position.
-  RC<RawSyntax>
+  RC<RawSyntax> replacingChild(CursorIndex Index,
-  replaceChild(CursorIndex Index, RC<RawSyntax> NewLayoutElement) const;
+                               RC<RawSyntax> NewLayoutElement) const;
 
   /// @}
 
-  /// Advance the provided AbsolutePosition by the full width of this node.
+  size_t getLeadingTriviaLength() {
-  ///
+    size_t Length = 0;
-  /// If this is token node, returns the AbsolutePosition of the start of the
+    for (auto Trivia : getLeadingTrivia()) {
-  /// token's nontrivial text. Otherwise, return the position of the first
+      Length += Trivia.getTextLength();
-  /// token. If this contains no tokens, return None.
+    }
-  llvm::Optional<AbsolutePosition>
+    return Length;
-  accumulateAbsolutePosition(AbsolutePosition &Pos) const;
+  }
 
-  /// Advance the provided AbsolutePosition by the first trivia of this node.
+  size_t getTrailingTriviaLength() {
-  /// Return true if we found this trivia; otherwise false.
+    size_t Length = 0;
-  bool accumulateLeadingTrivia(AbsolutePosition &Pos) const;
+    for (auto Trivia : getTrailingTrivia()) {
+      Length += Trivia.getTextLength();
+    }
+    return Length;
+  }
 
   /// Print this piece of syntax recursively.
   void print(llvm::raw_ostream &OS, SyntaxPrintOptions Opts) const;
@@ -584,8 +569,5 @@ public:
 } // end namespace syntax
 } // end namespace swift
 
-namespace llvm {
-raw_ostream &operator<<(raw_ostream &OS, swift::syntax::AbsolutePosition Pos);
-} // end namespace llvm
 
 #endif // SWIFT_SYNTAX_RAWSYNTAX_H

include/swift/Syntax/Serialization/SyntaxDeserialization.h

@@ -162,7 +162,7 @@ template <> struct MappingTraits<swift::RC<swift::RawSyntax>> {
       StringRef nodeIdString;
       in.mapRequired("id", nodeIdString);
       unsigned nodeId = std::atoi(nodeIdString.data());
-      value = swift::RawSyntax::make(
+      value = swift::RawSyntax::makeAndCalcLength(
           tokenKind, swift::OwnedString::makeRefCounted(text), leadingTrivia,
           trailingTrivia, presence, /*Arena=*/nullptr, nodeId);
     } else {
@@ -178,8 +178,8 @@ template <> struct MappingTraits<swift::RC<swift::RawSyntax>> {
       StringRef nodeIdString;
       in.mapRequired("id", nodeIdString);
       unsigned nodeId = std::atoi(nodeIdString.data());
-      value = swift::RawSyntax::make(kind, layout, presence, /*Arena=*/nullptr,
+      value = swift::RawSyntax::makeAndCalcLength(kind, layout, presence,
-                                     nodeId);
+                                                  /*Arena=*/nullptr, nodeId);
     }
   }
 };
@@ -198,7 +198,7 @@ public:
   llvm::Optional<swift::SourceFileSyntax> getSourceFileSyntax() {
     swift::RC<swift::RawSyntax> raw;
     Input >> raw;
-    return swift::make<swift::SourceFileSyntax>(raw);
+    return swift::makeRoot<swift::SourceFileSyntax>(raw);
   }
 };
 } // namespace json

include/swift/Syntax/Syntax.h

@@ -39,9 +39,8 @@ namespace syntax {
 struct SyntaxVisitor;
 class SourceFileSyntax;
 
-template <typename SyntaxNode>
+template <typename SyntaxNode> SyntaxNode makeRoot(RC<RawSyntax> Raw) {
-SyntaxNode make(RC<RawSyntax> Raw) {
+  auto Data = SyntaxData::make(AbsoluteRawSyntax::forRoot(Raw));
-  auto Data = SyntaxData::make(Raw);
   return { Data, Data.get() };
 }
 
@@ -80,7 +79,7 @@ public:
   SyntaxKind getKind() const;
 
   /// Get the shared raw syntax.
-  const RC<RawSyntax> &getRaw() const;
+  const RC<RawSyntax> getRaw() const;
 
   /// Get an ID for this node that is stable across incremental parses
   SyntaxNodeId getId() const { return getRaw()->getId(); }
@@ -192,23 +191,32 @@ public:
   /// Recursively visit this node.
   void accept(SyntaxVisitor &Visitor);
 
-  /// Get the absolute position of this raw syntax: its offset, line,
+  /// Same as \c getAbsolutePositionAfterLeadingTrivia.
-  /// and column.
+  AbsoluteOffsetPosition getAbsolutePosition() const {
-  AbsolutePosition getAbsolutePosition() const {
+    return getAbsolutePositionAfterLeadingTrivia();
-    return Data->getAbsolutePosition();
   }
 
-  /// Get the absolute end position (exclusively) where the trailing trivia of
+  /// Get the offset at which the leading trivia of this node starts.
-  /// this node ends.
+  AbsoluteOffsetPosition getAbsolutePositionBeforeLeadingTrivia() const {
-  AbsolutePosition getAbsoluteEndPositionAfterTrailingTrivia() const {
-    return Data->getAbsoluteEndPositionAfterTrailingTrivia();
-  }
-
-  /// Get the absolute position at which the leading trivia of this node starts.
-  AbsolutePosition getAbsolutePositionBeforeLeadingTrivia() const {
     return Data->getAbsolutePositionBeforeLeadingTrivia();
   }
 
+  /// Get the offset at which the actual content (i.e. non-triva) of this node
+  /// starts.
+  AbsoluteOffsetPosition getAbsolutePositionAfterLeadingTrivia() const {
+    return Data->getAbsolutePositionAfterLeadingTrivia();
+  }
+
+  /// Get the offset at which the trailing trivia of this node starts.
+  AbsoluteOffsetPosition getAbsoluteEndPositionBeforeTrailingTrivia() const {
+    return Data->getAbsoluteEndPositionBeforeTrailingTrivia();
+  }
+
+  /// Get the offset at which the trailing trivia of this node starts.
+  AbsoluteOffsetPosition getAbsoluteEndPositionAfterTrailingTrivia() const {
+    return Data->getAbsoluteEndPositionAfterTrailingTrivia();
+  }
+
   // TODO: hasSameStructureAs ?
 };
 

include/swift/Syntax/SyntaxCollection.h

@@ -63,9 +63,9 @@ private:
     List.reserve(Elements.size());
     for (auto &Elt : Elements)
       List.push_back(Elt.getRaw());
-    auto Raw = RawSyntax::make(CollectionKind, List,
+    auto Raw = RawSyntax::makeAndCalcLength(CollectionKind, List,
-                               SourcePresence::Present);
+                                            SourcePresence::Present);
-    return SyntaxData::make(Raw);
+    return SyntaxData::make(AbsoluteRawSyntax::forRoot(Raw));
   }
   SyntaxCollection(const RC<SyntaxData> Root): Syntax(Root, Root.get()) {}
 
@@ -120,7 +120,7 @@ public:
     std::copy(OldLayout.begin(), OldLayout.end(), back_inserter(NewLayout));
     NewLayout.push_back(E.getRaw());
     auto Raw = RawSyntax::make(CollectionKind, NewLayout, getRaw()->getPresence());
-    return Data->replaceSelf<SyntaxCollection<CollectionKind, Element>>(Raw);
+    return Data->replacingSelf<SyntaxCollection<CollectionKind, Element>>(Raw);
   }
 
   /// Return a new collection with an element removed from the end.
@@ -130,7 +130,7 @@ public:
     assert(!empty());
     auto NewLayout = getRaw()->getLayout().drop_back();
     auto Raw = RawSyntax::make(CollectionKind, NewLayout, getRaw()->getPresence());
-    return Data->replaceSelf<SyntaxCollection<CollectionKind, Element>>(Raw);
+    return Data->replacingSelf<SyntaxCollection<CollectionKind, Element>>(Raw);
   }
 
   /// Return a new collection with the given element appended to the front.
@@ -141,7 +141,7 @@ public:
     std::copy(OldLayout.begin(), OldLayout.end(),
               std::back_inserter(NewLayout));
     auto Raw = RawSyntax::make(CollectionKind, NewLayout, getRaw()->getPresence());
-    return Data->replaceSelf<SyntaxCollection<CollectionKind, Element>>(Raw);
+    return Data->replacingSelf<SyntaxCollection<CollectionKind, Element>>(Raw);
   }
 
   /// Return a new collection with an element removed from the end.
@@ -151,7 +151,7 @@ public:
     assert(!empty());
     auto NewLayout = getRaw()->getLayout().drop_front();
     auto Raw = RawSyntax::make(CollectionKind, NewLayout, getRaw()->getPresence());
-    return Data->replaceSelf<SyntaxCollection<CollectionKind, Element>>(Raw);
+    return Data->replacingSelf<SyntaxCollection<CollectionKind, Element>>(Raw);
   }
 
   /// Return a new collection with the Element inserted at index i.
@@ -170,7 +170,7 @@ public:
     std::copy(OldLayout.begin() + i, OldLayout.end(),
               std::back_inserter(NewLayout));
     auto Raw = RawSyntax::make(CollectionKind, NewLayout, getRaw()->getPresence());
-    return Data->replaceSelf<SyntaxCollection<CollectionKind, Element>>(Raw);
+    return Data->replacingSelf<SyntaxCollection<CollectionKind, Element>>(Raw);
   }
 
   /// Return a new collection with the element removed at index i.
@@ -181,13 +181,13 @@ public:
     std::advance(iterator, i);
     NewLayout.erase(iterator);
     auto Raw = RawSyntax::make(CollectionKind, NewLayout, getRaw()->getPresence());
-    return Data->replaceSelf<SyntaxCollection<CollectionKind, Element>>(Raw);
+    return Data->replacingSelf<SyntaxCollection<CollectionKind, Element>>(Raw);
   }
 
   /// Return an empty syntax collection of this type.
   SyntaxCollection<CollectionKind, Element> cleared() const {
     auto Raw = RawSyntax::make(CollectionKind, {}, getRaw()->getPresence());
-    return Data->replaceSelf<SyntaxCollection<CollectionKind, Element>>(Raw);
+    return Data->replacingSelf<SyntaxCollection<CollectionKind, Element>>(Raw);
   }
 
   static bool kindof(SyntaxKind Kind) {

include/swift/Syntax/SyntaxData.h

@@ -40,6 +40,7 @@
 #define SWIFT_SYNTAX_SYNTAXDATA_H
 
 #include "swift/Basic/Debug.h"
+#include "swift/Syntax/AbsoluteRawSyntax.h"
 #include "swift/Syntax/AtomicCache.h"
 #include "swift/Syntax/RawSyntax.h"
 #include "swift/Syntax/References.h"
@@ -55,111 +56,72 @@ namespace syntax {
 /// This structure should not contain significant public
 /// API or internal modification API.
 ///
-/// This is only for holding a strong reference to the RawSyntax, a weak
+/// It is essentially a wrapper around \c AbsoluteRawSyntax that also keeps
-/// reference to the parent, and, in subclasses, lazily created strong
+/// track of the parent.
-/// references to non-terminal child nodes.
+class SyntaxData final : public llvm::ThreadSafeRefCountedBase<SyntaxData> {
-class SyntaxData final
-    : public llvm::ThreadSafeRefCountedBase<SyntaxData>,
-      private llvm::TrailingObjects<SyntaxData, AtomicCache<SyntaxData>> {
-  friend TrailingObjects;
 
   using RootDataPair = std::pair<RC<SyntaxData>, RC<SyntaxData>>;
 
-  /// The shared raw syntax representing this syntax data node.
+  const AbsoluteRawSyntax AbsoluteRaw;
-  const RC<RawSyntax> Raw;
+  const RC<SyntaxData> Parent;
 
-  /// The parent of this syntax.
+  SyntaxData(AbsoluteRawSyntax AbsoluteRaw, const RC<SyntaxData> &Parent)
-  ///
+      : AbsoluteRaw(AbsoluteRaw), Parent(Parent) {}
-  /// WARNING! Do not access this directly. Use getParent(),
-  /// which enforces nullptr checking.
-  const SyntaxData *Parent;
 
-  /// The index into the parent's child layout.
+  /// With a new \c RawSyntax node, create a new node from this one and
-  ///
-  /// If there is no parent, this is 0.
-  const CursorIndex IndexInParent;
-
-  /// Cache the absolute position of this node.
-  Optional<AbsolutePosition> PositionCache;
-
-  size_t numTrailingObjects(OverloadToken<AtomicCache<SyntaxData>>) const {
-    return Raw->getNumChildren();
-  }
-
-  SyntaxData(RC<RawSyntax> Raw, const SyntaxData *Parent = nullptr,
-             CursorIndex IndexInParent = 0)
-      : Raw(Raw), Parent(Parent), IndexInParent(IndexInParent) {
-    auto *I = getTrailingObjects<AtomicCache<SyntaxData>>();
-    for (auto *E = I + getNumChildren(); I != E; ++I)
-      ::new (static_cast<void *>(I)) AtomicCache<SyntaxData>();
-  }
-
-  /// With a new RawSyntax node, create a new node from this one and
   /// recursively rebuild the parental chain up to the root.
   ///
   /// DO NOT expose this as public API.
-  RootDataPair replaceSelf(const RC<RawSyntax> NewRaw) const {
+  RootDataPair replacingSelf(const RC<RawSyntax> &NewRaw) const {
     if (hasParent()) {
-      auto NewRootAndParent = Parent->replaceChild(NewRaw, IndexInParent);
+      auto NewRootAndParent =
-      auto NewMe = NewRootAndParent.second->getChild(IndexInParent);
+          Parent->replacingChild(NewRaw, getIndexInParent());
+      auto NewMe = NewRootAndParent.second->getChild(getIndexInParent());
       return { NewRootAndParent.first, NewMe.get() };
     } else {
-      auto NewMe = make(NewRaw, nullptr, IndexInParent);
+      auto NewMe = make(AbsoluteRawSyntax::forRoot(NewRaw), nullptr);
       return { NewMe, NewMe.get() };
     }
   }
 
-  /// Create the data for a child node with the raw syntax in our layout
-  /// at the provided index.
-  /// DO NOT expose this as public API.
-  RC<SyntaxData> realizeSyntaxNode(CursorIndex Index) const {
-    if (auto &RawChild = Raw->getChild(Index))
-      return SyntaxData::make(RawChild, this, Index);
-    return nullptr;
-  }
-
   /// Replace a child in the raw syntax and recursively rebuild the
   /// parental chain up to the root.
   ///
   /// DO NOT expose this as public API.
   template <typename CursorType>
-  RootDataPair replaceChild(const RC<RawSyntax> RawChild,
+  RootDataPair replacingChild(const RC<RawSyntax> &RawChild,
-                            CursorType ChildCursor) const {
+                              CursorType ChildCursor) const {
-    auto NewRaw = Raw->replaceChild(ChildCursor, RawChild);
+    auto NewRaw = AbsoluteRaw.getRaw()->replacingChild(ChildCursor, RawChild);
-    return replaceSelf(NewRaw);
+    return replacingSelf(NewRaw);
   }
 
-  ArrayRef<AtomicCache<SyntaxData>> getChildren() const {
+  RC<SyntaxData> getRefCountedThis() const {
-    return {getTrailingObjects<AtomicCache<SyntaxData>>(), getNumChildren()};
+    // FIXME: Is the usage of const_cast safe here?
+    return RC<SyntaxData>(const_cast<SyntaxData *>(this));
   }
 
 public:
-  /// Disable sized deallocation for SyntaxData, because it has tail-allocated
-  /// data.
-  void operator delete(void *p) { ::operator delete(p); }
-
   /// Get the node immediately before this current node that does contain a
-  /// non-missing token. Return nullptr if we cannot find such node.
+  /// non-missing token. Return \c nullptr if we cannot find such node.
   RC<SyntaxData> getPreviousNode() const;
 
   /// Get the node immediately after this current node that does contain a
-  /// non-missing token. Return nullptr if we cannot find such node.
+  /// non-missing token. Return \c nullptr if we cannot find such node.
   RC<SyntaxData> getNextNode() const;
 
-  /// Get the first non-missing token node in this tree. Return nullptr if this
+  /// Get the first non-missing token node in this tree. Return \c nullptr if
-  /// node does not contain non-missing tokens.
+  /// this node does not contain non-missing tokens.
   RC<SyntaxData> getFirstToken() const;
 
-  ~SyntaxData() {
+  /// Get the last non-missing token node in this tree. Return \c nullptr if
-    for (auto &I : getChildren())
+  /// this node does not contain non-missing tokens.
-      I.~AtomicCache<SyntaxData>();
+  RC<SyntaxData> getLastToken() const;
-  }
 
   /// Constructs a SyntaxNode by replacing `self` and recursively building
   /// the parent chain up to the root.
   template <typename SyntaxNode>
-  SyntaxNode replaceSelf(const RC<RawSyntax> NewRaw) const {
+  SyntaxNode replacingSelf(const RC<RawSyntax> &NewRaw) const {
-    auto NewRootAndData = replaceSelf(NewRaw);
+    auto NewRootAndData = replacingSelf(NewRaw);
     return { NewRootAndData.first, NewRootAndData.second.get() };
   }
 
@@ -168,34 +130,30 @@ public:
   ///
   /// DO NOT expose this as public API.
   template <typename SyntaxNode, typename CursorType>
-  SyntaxNode replaceChild(const RC<RawSyntax> RawChild,
+  SyntaxNode replacingChild(const RC<RawSyntax> &RawChild,
-                          CursorType ChildCursor) const {
+                            CursorType ChildCursor) const {
-    auto NewRootAndParent = replaceChild(RawChild, ChildCursor);
+    auto NewRootAndParent = replacingChild(RawChild, ChildCursor);
     return SyntaxNode {
       NewRootAndParent.first,
       NewRootAndParent.second.get()
     };
   }
 
-
+  /// Make a new \c SyntaxData node for the tree's root.
-  static RC<SyntaxData> make(RC<RawSyntax> Raw,
+  static RC<SyntaxData> make(AbsoluteRawSyntax AbsoluteRaw) {
-                             const SyntaxData *Parent = nullptr,
+    return make(AbsoluteRaw, nullptr);
-                             CursorIndex IndexInParent = 0);
+  }
+  static RC<SyntaxData> make(AbsoluteRawSyntax AbsoluteRaw,
+                             const RC<SyntaxData> &Parent);
 
   /// Returns the raw syntax node for this syntax node.
-  const RC<RawSyntax> &getRaw() const {
+  const RC<RawSyntax> &getRaw() const { return AbsoluteRaw.getRaw(); }
-    return Raw;
-  }
 
   /// Returns the kind of syntax node this is.
-  SyntaxKind getKind() const {
+  SyntaxKind getKind() const { return AbsoluteRaw.getRaw()->getKind(); }
-    return Raw->getKind();
-  }
 
   /// Return the parent syntax if there is one.
-  const SyntaxData * getParent() const {
+  const RC<SyntaxData> &getParent() const { return Parent; }
-    return Parent;
-  }
 
   /// Returns true if this syntax node has a parent.
   bool hasParent() const {
@@ -204,20 +162,21 @@ public:
 
   /// Returns the child index of this node in its parent, if it has a parent,
   /// otherwise 0.
-  size_t getIndexInParent() const {
+  AbsoluteSyntaxPosition::IndexInParentType getIndexInParent() const {
-    return IndexInParent;
+    return AbsoluteRaw.getPosition().getIndexInParent();
   }
 
   /// Returns the number of children this SyntaxData represents.
   size_t getNumChildren() const {
-    return Raw->getLayout().size();
+    return AbsoluteRaw.getRaw()->getLayout().size();
   }
 
   /// Gets the child at the index specified by the provided cursor,
   /// lazily creating it if necessary.
   template <typename CursorType>
   RC<SyntaxData> getChild(CursorType Cursor) const {
-    return getChild((size_t)cursorIndex(Cursor));
+    return getChild(
+        (AbsoluteSyntaxPosition::IndexInParentType)cursorIndex(Cursor));
   }
 
   /// Gets the child at the specified index in this data's children array.
@@ -247,25 +206,22 @@ public:
   /// then we ask the AtomicCache in that position to realize its value and
   /// cache it. This is safe because AtomicCache only ever mutates its cache
   /// one time -- the first initialization that wins a compare_exchange_strong.
-  RC<SyntaxData> getChild(size_t Index) const {
+  RC<SyntaxData>
-    if (!getRaw()->getChild(Index))
+  getChild(AbsoluteSyntaxPosition::IndexInParentType Index) const;
-      return nullptr;
-    return getChildren()[Index].getOrCreate([&]() {
-      return realizeSyntaxNode(Index);
-    });
-  }
 
-  /// Calculate the absolute position of this node, use cache if the cache
+  /// Get the offset at which the leading trivia of this node starts.
-  /// is populated.
+  AbsoluteOffsetPosition getAbsolutePositionBeforeLeadingTrivia() const;
-  AbsolutePosition getAbsolutePosition() const;
 
-  /// Calculate the absolute end position of this node, use cache of the immediate
+  /// Get the offset at which the content (excluding trailing trivia) of this
-  /// next node if populated.
+  /// node ends.
-  AbsolutePosition getAbsoluteEndPositionAfterTrailingTrivia() const;
+  AbsoluteOffsetPosition getAbsoluteEndPositionBeforeTrailingTrivia() const;
 
-  /// Get the absolute position without skipping the leading trivia of this
+  /// Get the offset at which the content of this node (excluding leading
-  /// node.
+  /// trivia) starts.
-  AbsolutePosition getAbsolutePositionBeforeLeadingTrivia() const;
+  AbsoluteOffsetPosition getAbsolutePositionAfterLeadingTrivia() const;
+
+  /// Get the offset at chiwh the trailing trivia of this node ends.
+  AbsoluteOffsetPosition getAbsoluteEndPositionAfterTrailingTrivia() const;
 
   /// Returns true if the data node represents type syntax.
   bool isType() const;

include/swift/Syntax/TokenSyntax.h

@@ -37,7 +37,7 @@ public:
     : Syntax(Root, Data) {}
 
   static TokenSyntax missingToken(const tok Kind, OwnedString Text) {
-    return make<TokenSyntax>(RawSyntax::missing(Kind, Text));
+    return makeRoot<TokenSyntax>(RawSyntax::missing(Kind, Text));
   }
 
   Trivia getLeadingTrivia() const {
@@ -50,12 +50,12 @@ public:
 
   TokenSyntax withLeadingTrivia(const Trivia &Trivia) const {
     auto NewRaw = getRaw()->withLeadingTrivia(Trivia.Pieces);
-    return Data->replaceSelf<TokenSyntax>(NewRaw);
+    return Data->replacingSelf<TokenSyntax>(NewRaw);
   }
 
   TokenSyntax withTrailingTrivia(const Trivia &Trivia) const {
     auto NewRaw = getRaw()->withTrailingTrivia(Trivia.Pieces);
-    return Data->replaceSelf<TokenSyntax>(NewRaw);
+    return Data->replacingSelf<TokenSyntax>(NewRaw);
   }
 
   /* TODO: If we really need them.

include/swift/Syntax/Trivia.h.gyb

@@ -99,8 +99,6 @@
 namespace swift {
 namespace syntax {
 
-class AbsolutePosition;
-
 /// The kind of source trivia, such as spaces, newlines, or comments.
 enum class TriviaKind {
 % for trivia in TRIVIAS:
@@ -181,8 +179,6 @@ public:
   bool isComment() const {
     return isCommentTriviaKind(getKind());
   }
-
-  void accumulateAbsolutePosition(AbsolutePosition &Pos) const;
   
   /// Try to compose this and Next to one TriviaPiece.
   /// It returns true if it is succeeded.

lib/Parse/Parser.cpp

@@ -320,14 +320,13 @@ std::vector<Token> swift::tokenize(const LangOptions &LangOpts,
   return Tokens;
 }
 
-std::vector<std::pair<RC<syntax::RawSyntax>, syntax::AbsolutePosition>>
+std::vector<std::pair<RC<syntax::RawSyntax>, syntax::AbsoluteOffsetPosition>>
 swift::tokenizeWithTrivia(const LangOptions &LangOpts, const SourceManager &SM,
                           unsigned BufferID, unsigned Offset,
-                          unsigned EndOffset,
+                          unsigned EndOffset, DiagnosticEngine *Diags) {
-                          DiagnosticEngine *Diags) {
+  std::vector<std::pair<RC<syntax::RawSyntax>, syntax::AbsoluteOffsetPosition>>
-  std::vector<std::pair<RC<syntax::RawSyntax>, syntax::AbsolutePosition>>
       Tokens;
-  syntax::AbsolutePosition RunningPos;
+  syntax::AbsoluteOffsetPosition RunningPos(0);
 
   tokenize(
       LangOpts, SM, BufferID, Offset, EndOffset, Diags,
@@ -346,13 +345,17 @@ swift::tokenizeWithTrivia(const LangOptions &LangOpts, const SourceManager &SM,
         Trivia syntaxTrailingTrivia =
           TrailingTrivia.convertToSyntaxTrivia(TrailingTriviaLoc, SM, BufferID);
         auto Text = OwnedString::makeRefCounted(Tok.getRawText());
-        auto ThisToken =
+        size_t TextLength = LeadingTrivia.getLength() +
-            RawSyntax::make(Tok.getKind(), Text, syntaxLeadingTrivia.Pieces,
+                            TokRange.getByteLength() +
-                            syntaxTrailingTrivia.Pieces,
+                            TrailingTrivia.getLength();
-                            SourcePresence::Present);
+        auto ThisToken = RawSyntax::make(
+            Tok.getKind(), Text, TextLength, syntaxLeadingTrivia.Pieces,
+            syntaxTrailingTrivia.Pieces, SourcePresence::Present);
 
-        auto ThisTokenPos = ThisToken->accumulateAbsolutePosition(RunningPos);
+        auto ThisTokenPos =
-        Tokens.push_back({ThisToken, ThisTokenPos.getValue()});
+            RunningPos.advancedBy(ThisToken->getLeadingTriviaLength());
+        Tokens.push_back({ThisToken, ThisTokenPos});
+        RunningPos = RunningPos.advancedBy(ThisToken->getTextLength());
       });
 
   return Tokens;

lib/Syntax/AbsoluteRawSyntax.cpp

@@ -0,0 +1,67 @@
+//===--- AbsoluteRawSyntax.cpp ----------------------------------*- C++ -*-===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+#include "swift/Syntax/AbsoluteRawSyntax.h"
+
+using namespace swift;
+using namespace swift::syntax;
+
+std::atomic<SyntaxIdentifier::RootIdType> SyntaxIdentifier::NextUnusedRootId(0);
+
+SyntaxIndexInTree
+SyntaxIndexInTree::advancedBy(const RC<RawSyntax> &Raw) const {
+  auto NewIndexInTree = IndexInTree;
+  if (Raw) {
+    NewIndexInTree += Raw->getTotalNodes();
+  }
+  return SyntaxIndexInTree(NewIndexInTree);
+}
+
+SyntaxIndexInTree
+SyntaxIndexInTree::reversedBy(const RC<RawSyntax> &Raw) const {
+  auto NewIndexInTree = IndexInTree;
+  if (Raw) {
+    NewIndexInTree -= Raw->getTotalNodes();
+  }
+  return SyntaxIndexInTree(NewIndexInTree);
+}
+
+SyntaxIndexInTree SyntaxIndexInTree::advancedToFirstChild() const {
+  auto NewIndexInTree = IndexInTree + 1;
+  return SyntaxIndexInTree(NewIndexInTree);
+}
+
+AbsoluteSyntaxPosition
+AbsoluteSyntaxPosition::advancedBy(const RC<RawSyntax> &Raw) const {
+  OffsetType NewOffset = Offset;
+  if (Raw) {
+    NewOffset += Raw->getTextLength();
+  }
+  IndexInParentType NewIndexInParent = IndexInParent + 1;
+  return AbsoluteSyntaxPosition(NewOffset, NewIndexInParent);
+}
+
+AbsoluteSyntaxPosition
+AbsoluteSyntaxPosition::reversedBy(const RC<RawSyntax> &Raw) const {
+  OffsetType NewOffset = Offset;
+  if (Raw) {
+    NewOffset -= Raw->getTextLength();
+  }
+  IndexInParentType NewIndexInParent = IndexInParent - 1;
+  return AbsoluteSyntaxPosition(NewOffset, NewIndexInParent);
+}
+
+raw_ostream &llvm::operator<<(raw_ostream &OS,
+                              swift::syntax::AbsoluteOffsetPosition Pos) {
+  OS << "Offset " << Pos.getOffset();
+  return OS;
+}

lib/Syntax/CMakeLists.txt

@@ -5,6 +5,7 @@ else()
 endif()
 
 add_swift_host_library(swiftSyntax STATIC
+  AbsoluteRawSyntax.cpp
   RawSyntax.cpp
   Syntax.cpp
   SyntaxData.cpp

lib/Syntax/RawSyntax.cpp

@@ -67,26 +67,37 @@ void swift::dumpTokenKind(llvm::raw_ostream &OS, tok Kind) {
   }
 }
 
+// FIXME: If we want thread-safety for tree creation, this needs to be atomic.
 unsigned RawSyntax::NextFreeNodeId = 1;
 
 RawSyntax::RawSyntax(SyntaxKind Kind, ArrayRef<RC<RawSyntax>> Layout,
-                     SourcePresence Presence, const RC<SyntaxArena> &Arena,
+                     size_t TextLength, SourcePresence Presence,
+                     const RC<SyntaxArena> &Arena,
                      llvm::Optional<unsigned> NodeId) {
   assert(Kind != SyntaxKind::Token &&
          "'token' syntax node must be constructed with dedicated constructor");
 
   RefCount = 0;
 
+  size_t TotalSubNodeCount = 0;
+  for (auto Child : Layout) {
+    if (Child) {
+      TotalSubNodeCount += Child->getTotalSubNodeCount() + 1;
+    }
+  }
+
   if (NodeId.hasValue()) {
     this->NodeId = NodeId.getValue();
     NextFreeNodeId = std::max(this->NodeId + 1, NextFreeNodeId);
   } else {
     this->NodeId = NextFreeNodeId++;
   }
-  Bits.Common.Kind = unsigned(Kind);
+  Bits.Common.TextLength = TextLength;
   Bits.Common.Presence = unsigned(Presence);
+  Bits.Common.IsToken = false;
   Bits.Layout.NumChildren = Layout.size();
-  Bits.Layout.TextLength = UINT32_MAX;
+  Bits.Layout.TotalSubNodeCount = TotalSubNodeCount;
+  Bits.Layout.Kind = unsigned(Kind);
 
   this->Arena = Arena;
 
@@ -95,7 +106,7 @@ RawSyntax::RawSyntax(SyntaxKind Kind, ArrayRef<RC<RawSyntax>> Layout,
                           getTrailingObjects<RC<RawSyntax>>());
 }
 
-RawSyntax::RawSyntax(tok TokKind, OwnedString Text,
+RawSyntax::RawSyntax(tok TokKind, OwnedString Text, size_t TextLength,
                      ArrayRef<TriviaPiece> LeadingTrivia,
                      ArrayRef<TriviaPiece> TrailingTrivia,
                      SourcePresence Presence, const RC<SyntaxArena> &Arena,
@@ -108,8 +119,9 @@ RawSyntax::RawSyntax(tok TokKind, OwnedString Text,
   } else {
     this->NodeId = NextFreeNodeId++;
   }
-  Bits.Common.Kind = unsigned(SyntaxKind::Token);
+  Bits.Common.TextLength = TextLength;
   Bits.Common.Presence = unsigned(Presence);
+  Bits.Common.IsToken = true;
   Bits.Token.TokenKind = unsigned(TokKind);
   Bits.Token.NumLeadingTrivia = LeadingTrivia.size();
   Bits.Token.NumTrailingTrivia = TrailingTrivia.size();
@@ -142,7 +154,7 @@ RawSyntax::~RawSyntax() {
 }
 
 RC<RawSyntax> RawSyntax::make(SyntaxKind Kind, ArrayRef<RC<RawSyntax>> Layout,
-                              SourcePresence Presence,
+                              size_t TextLength, SourcePresence Presence,
                               const RC<SyntaxArena> &Arena,
                               llvm::Optional<unsigned> NodeId) {
   auto size = totalSizeToAlloc<RC<RawSyntax>, OwnedString, TriviaPiece>(
@@ -150,10 +162,10 @@ RC<RawSyntax> RawSyntax::make(SyntaxKind Kind, ArrayRef<RC<RawSyntax>> Layout,
   void *data = Arena ? Arena->Allocate(size, alignof(RawSyntax))
                      : ::operator new(size);
   return RC<RawSyntax>(
-      new (data) RawSyntax(Kind, Layout, Presence, Arena, NodeId));
+      new (data) RawSyntax(Kind, Layout, TextLength, Presence, Arena, NodeId));
 }
 
-RC<RawSyntax> RawSyntax::make(tok TokKind, OwnedString Text,
+RC<RawSyntax> RawSyntax::make(tok TokKind, OwnedString Text, size_t TextLength,
                               ArrayRef<TriviaPiece> LeadingTrivia,
                               ArrayRef<TriviaPiece> TrailingTrivia,
                               SourcePresence Presence,
@@ -163,9 +175,9 @@ RC<RawSyntax> RawSyntax::make(tok TokKind, OwnedString Text,
       0, 1, LeadingTrivia.size() + TrailingTrivia.size());
   void *data = Arena ? Arena->Allocate(size, alignof(RawSyntax))
                      : ::operator new(size);
-  return RC<RawSyntax>(new (data) RawSyntax(TokKind, Text, LeadingTrivia,
+  return RC<RawSyntax>(new (data)
-                                            TrailingTrivia, Presence,
+                           RawSyntax(TokKind, Text, TextLength, LeadingTrivia,
-                                            Arena, NodeId));
+                                     TrailingTrivia, Presence, Arena, NodeId));
 }
 
 RC<RawSyntax> RawSyntax::append(RC<RawSyntax> NewLayoutElement) const {
@@ -174,11 +186,12 @@ RC<RawSyntax> RawSyntax::append(RC<RawSyntax> NewLayoutElement) const {
   NewLayout.reserve(Layout.size() + 1);
   std::copy(Layout.begin(), Layout.end(), std::back_inserter(NewLayout));
   NewLayout.push_back(NewLayoutElement);
-  return RawSyntax::make(getKind(), NewLayout, SourcePresence::Present);
+  return RawSyntax::makeAndCalcLength(getKind(), NewLayout,
+                                      SourcePresence::Present);
 }
 
-RC<RawSyntax> RawSyntax::replaceChild(CursorIndex Index,
+RC<RawSyntax> RawSyntax::replacingChild(CursorIndex Index,
-                                      RC<RawSyntax> NewLayoutElement) const {
+                                        RC<RawSyntax> NewLayoutElement) const {
   auto Layout = getLayout();
   std::vector<RC<RawSyntax>> NewLayout;
   NewLayout.reserve(Layout.size());
@@ -191,49 +204,7 @@ RC<RawSyntax> RawSyntax::replaceChild(CursorIndex Index,
   std::copy(Layout.begin() + Index + 1, Layout.end(),
             std::back_inserter(NewLayout));
 
-  return RawSyntax::make(getKind(), NewLayout, getPresence());
+  return RawSyntax::makeAndCalcLength(getKind(), NewLayout, getPresence());
-}
-
-llvm::Optional<AbsolutePosition>
-RawSyntax::accumulateAbsolutePosition(AbsolutePosition &Pos) const {
-  llvm::Optional<AbsolutePosition> Ret;
-  if (isToken()) {
-    if (isMissing())
-      return None;
-    for (auto &Leader : getLeadingTrivia())
-      Leader.accumulateAbsolutePosition(Pos);
-    Ret = Pos;
-    Pos.addText(getTokenText());
-    for (auto &Trailer : getTrailingTrivia())
-      Trailer.accumulateAbsolutePosition(Pos);
-  } else {
-    for (auto &Child : getLayout()) {
-      if (!Child)
-        continue;
-      auto Result = Child->accumulateAbsolutePosition(Pos);
-      if (!Ret && Result)
-        Ret = Result;
-    }
-  }
-  return Ret;
-}
-
-bool RawSyntax::accumulateLeadingTrivia(AbsolutePosition &Pos) const {
- if (isToken()) {
-    if (!isMissing()) {
-      for (auto &Leader: getLeadingTrivia())
-        Leader.accumulateAbsolutePosition(Pos);
-      return true;
-    }
-  } else {
-    for (auto &Child: getLayout()) {
-      if (!Child || Child->isMissing())
-        continue;
-      if (Child->accumulateLeadingTrivia(Pos))
-        return true;
-    }
-  }
-  return false;
 }
 
 void RawSyntax::print(llvm::raw_ostream &OS, SyntaxPrintOptions Opts) const {
@@ -313,22 +284,6 @@ void RawSyntax::dump(llvm::raw_ostream &OS, unsigned Indent) const {
   OS << ')';
 }
 
-void AbsolutePosition::printLineAndColumn(llvm::raw_ostream &OS) const {
-  OS << getLine() << ':' << getColumn();
-}
-
-void AbsolutePosition::dump(llvm::raw_ostream &OS) const {
-  OS << "(absolute_position ";
-  OS << "offset=" << getOffset() << " ";
-  OS << "line=" << getLine() << " ";
-  OS << "column=" << getColumn();
-  OS << ')';
-}
-
-void AbsolutePosition::dump() const {
-  dump(llvm::errs());
-}
-
 void RawSyntax::Profile(llvm::FoldingSetNodeID &ID, tok TokKind,
                         OwnedString Text, ArrayRef<TriviaPiece> LeadingTrivia,
                         ArrayRef<TriviaPiece> TrailingTrivia) {
@@ -351,8 +306,3 @@ void RawSyntax::Profile(llvm::FoldingSetNodeID &ID, tok TokKind,
   for (auto &Piece : TrailingTrivia)
     Piece.Profile(ID);
 }
-
-llvm::raw_ostream &llvm::operator<<(raw_ostream &OS, AbsolutePosition Pos) {
-  Pos.printLineAndColumn(OS);
-  return OS;
-}

lib/Syntax/Syntax.cpp

@@ -17,9 +17,7 @@
 using namespace swift;
 using namespace swift::syntax;
 
-const RC<RawSyntax> &Syntax::getRaw() const {
+const RC<RawSyntax> Syntax::getRaw() const { return Data->getRaw(); }
-  return Data->getRaw();
-}
 
 SyntaxKind Syntax::getKind() const {
   return getRaw()->getKind();
@@ -77,9 +75,7 @@ bool Syntax::isMissing() const {
 llvm::Optional<Syntax> Syntax::getParent() const {
   auto ParentData = getData().getParent();
   if (!ParentData) return llvm::None;
-  return llvm::Optional<Syntax> {
+  return llvm::Optional<Syntax>{Syntax{Root, ParentData.get()}};
-    Syntax { Root, ParentData }
-  };
 }
 
 Syntax Syntax::getRoot() const {

lib/Syntax/SyntaxBuilders.cpp.gyb

@@ -45,9 +45,9 @@ ${node.name}Builder &
 ${node.name}Builder::add${child_elt}(${child_elt_type} ${child_elt}) {
   auto &raw = Layout[cursorIndex(${node.name}::Cursor::${child.name})];
   if (!raw)
-    raw = RawSyntax::make(SyntaxKind::${child_node.syntax_kind},
+    raw = RawSyntax::makeAndCalcLength(SyntaxKind::${child_node.syntax_kind},
-                          {${child_elt}.getRaw()},
+                                       {${child_elt}.getRaw()},
-                          SourcePresence::Present, Arena);
+                                       SourcePresence::Present, Arena);
   else
     raw = raw->append(${child_elt}.getRaw());
   return *this;
@@ -64,9 +64,9 @@ ${node.name}Builder::build() {
 %     end
 %   end
 % end
-  auto raw = RawSyntax::make(SyntaxKind::${node.syntax_kind}, 
+  auto raw = RawSyntax::makeAndCalcLength(SyntaxKind::${node.syntax_kind},
     Layout, SourcePresence::Present, Arena);
-  return make<${node.name}>(raw);
+  return makeRoot<${node.name}>(raw);
 }
 
 %   end

lib/Syntax/SyntaxData.cpp

@@ -15,40 +15,26 @@
 using namespace swift;
 using namespace swift::syntax;
 
-RC<SyntaxData> SyntaxData::make(RC<RawSyntax> Raw,
+RC<SyntaxData> SyntaxData::make(AbsoluteRawSyntax AbsoluteRaw,
-                                const SyntaxData *Parent,
+                                const RC<SyntaxData> &Parent) {
-                                CursorIndex IndexInParent) {
+  // FIXME: Can we use a bump allocator here?
-  auto size = totalSizeToAlloc<AtomicCache<SyntaxData>>(Raw->getNumChildren());
+  return RC<SyntaxData>{new SyntaxData(AbsoluteRaw, Parent)};
-  void *data = ::operator new(size);
-  return RC<SyntaxData>{new (data) SyntaxData(Raw, Parent, IndexInParent)};
 }
 
-bool SyntaxData::isType() const {
+bool SyntaxData::isType() const { return getRaw()->isType(); }
-  return Raw->isType();
-}
 
-bool SyntaxData::isStmt() const {
+bool SyntaxData::isStmt() const { return getRaw()->isStmt(); }
-  return Raw->isStmt();
-}
 
-bool SyntaxData::isDecl() const {
+bool SyntaxData::isDecl() const { return getRaw()->isDecl(); }
-  return Raw->isDecl();
-}
 
-bool SyntaxData::isExpr() const {
+bool SyntaxData::isExpr() const { return getRaw()->isExpr(); }
-  return Raw->isExpr();
-}
 
-bool SyntaxData::isPattern() const {
+bool SyntaxData::isPattern() const { return getRaw()->isPattern(); }
-  return Raw->isPattern();
-}
 
-bool SyntaxData::isUnknown() const {
+bool SyntaxData::isUnknown() const { return getRaw()->isUnknown(); }
-  return Raw->isUnknown();
-}
 
 void SyntaxData::dump(llvm::raw_ostream &OS) const {
-  Raw->dump(OS, 0);
+  getRaw()->dump(OS, 0);
   OS << '\n';
 }
 
@@ -86,9 +72,7 @@ RC<SyntaxData> SyntaxData::getNextNode() const {
 
 RC<SyntaxData> SyntaxData::getFirstToken() const {
   if (getRaw()->isToken()) {
-    // Get a reference counted version of this
+    return getRefCountedThis();
-    assert(hasParent() && "The syntax tree should not conisist only of the root");
-    return getParent()->getChild(getIndexInParent());
   }
 
   for (size_t I = 0, E = getNumChildren(); I < E; ++I) {
@@ -105,32 +89,83 @@ RC<SyntaxData> SyntaxData::getFirstToken() const {
   return nullptr;
 }
 
-AbsolutePosition SyntaxData::getAbsolutePositionBeforeLeadingTrivia() const {
+RC<SyntaxData> SyntaxData::getLastToken() const {
-  if (PositionCache.hasValue())
+  if (getRaw()->isToken() && !getRaw()->isMissing()) {
-    return *PositionCache;
+    return getRefCountedThis();
-  if (auto P = getPreviousNode()) {
-    auto Result = P->getAbsolutePositionBeforeLeadingTrivia();
-    P->getRaw()->accumulateAbsolutePosition(Result);
-    // FIXME: avoid using const_cast.
-    const_cast<SyntaxData*>(this)->PositionCache = Result;
-  } else {
-    const_cast<SyntaxData*>(this)->PositionCache = AbsolutePosition();
   }
-  return *PositionCache;
+
+  if (getNumChildren() == 0) {
+    return nullptr;
+  }
+  for (int I = getNumChildren() - 1; I >= 0; --I) {
+    if (auto Child = getChild(I)) {
+      if (Child->getRaw()->isMissing()) {
+        continue;
+      }
+      if (Child->getRaw()->isToken()) {
+        return Child;
+      } else if (auto Token = Child->getLastToken()) {
+        return Token;
+      }
+    }
+  }
+  return nullptr;
 }
 
-AbsolutePosition SyntaxData::getAbsolutePosition() const {
+RC<SyntaxData>
-  auto Result = getAbsolutePositionBeforeLeadingTrivia();
+SyntaxData::getChild(AbsoluteSyntaxPosition::IndexInParentType Index) const {
-  getRaw()->accumulateLeadingTrivia(Result);
+  if (!getRaw()->getChild(Index)) {
-  return Result;
+    return nullptr;
+  }
+  /// FIXME: Start from the back (advancedToEndOfChildren) and reverse from
+  /// there if Index is closer to the end as a performance improvement?
+  AbsoluteSyntaxPosition Position =
+      AbsoluteRaw.getInfo().getPosition().advancedToFirstChild();
+  SyntaxIdentifier NodeId =
+      AbsoluteRaw.getInfo().getNodeId().advancedToFirstChild();
+
+  for (size_t I = 0; I < Index; ++I) {
+    Position = Position.advancedBy(getRaw()->getChild(I));
+    NodeId = NodeId.advancedBy(getRaw()->getChild(I));
+  }
+  AbsoluteSyntaxInfo Info(Position, NodeId);
+
+  // FIXME: We are leaking here.
+  const RC<SyntaxData> RefCountedParent = getRefCountedThis();
+  RC<SyntaxData> Data = SyntaxData::make(
+      AbsoluteRawSyntax(getRaw()->getChild(Index), Info), RefCountedParent);
+  auto Data2 = Data;
+  Data2.resetWithoutRelease();
+  return Data;
 }
 
-AbsolutePosition SyntaxData::getAbsoluteEndPositionAfterTrailingTrivia() const {
+AbsoluteOffsetPosition
-  if (auto N = getNextNode()) {
+SyntaxData::getAbsolutePositionBeforeLeadingTrivia() const {
-    return N->getAbsolutePositionBeforeLeadingTrivia();
+  return AbsoluteRaw.getPosition();
+}
+
+AbsoluteOffsetPosition
+SyntaxData::getAbsolutePositionAfterLeadingTrivia() const {
+  if (auto FirstToken = getFirstToken()) {
+    return getAbsolutePositionBeforeLeadingTrivia().advancedBy(
+        FirstToken->getRaw()->getLeadingTriviaLength());
   } else {
-    auto Result = getAbsolutePositionBeforeLeadingTrivia();
+    return getAbsolutePositionBeforeLeadingTrivia();
-    getRaw()->accumulateAbsolutePosition(Result);
-    return Result;
   }
 }
+
+AbsoluteOffsetPosition
+SyntaxData::getAbsoluteEndPositionBeforeTrailingTrivia() const {
+  if (auto LastToken = getLastToken()) {
+    return getAbsoluteEndPositionAfterTrailingTrivia().advancedBy(
+        -LastToken->getRaw()->getTrailingTriviaLength());
+  } else {
+    return getAbsoluteEndPositionAfterTrailingTrivia();
+  }
+}
+
+AbsoluteOffsetPosition
+SyntaxData::getAbsoluteEndPositionAfterTrailingTrivia() const {
+  return getAbsolutePositionBeforeLeadingTrivia().advancedBy(
+      getRaw()->getTextLength());
+}

lib/Syntax/SyntaxFactory.cpp.gyb

@@ -43,9 +43,8 @@ TokenSyntax SyntaxFactory::makeToken(tok Kind, OwnedString Text,
                                      const Trivia &TrailingTrivia,
                                      SourcePresence Presence,
                                      RC<SyntaxArena> Arena) {
-  return make<TokenSyntax>(RawSyntax::make(Kind, Text, LeadingTrivia.Pieces,
+  return makeRoot<TokenSyntax>(RawSyntax::makeAndCalcLength(Kind, Text, 
-                                           TrailingTrivia.Pieces, Presence,
+    LeadingTrivia.Pieces, TrailingTrivia.Pieces, Presence, Arena));
-                                           Arena));
 }
 
 UnknownSyntax
@@ -56,9 +55,9 @@ SyntaxFactory::makeUnknownSyntax(llvm::ArrayRef<TokenSyntax> Tokens,
   for (auto &Token : Tokens) {
     Layout.push_back(Token.getRaw());
   }
-  auto Raw = RawSyntax::make(SyntaxKind::Unknown, Layout,
+  auto Raw = RawSyntax::makeAndCalcLength(SyntaxKind::Unknown, Layout,
-                             SourcePresence::Present, Arena);
+                                          SourcePresence::Present, Arena);
-  return make<UnknownSyntax>(Raw);
+  return makeRoot<UnknownSyntax>(Raw);
 }
 
 Syntax SyntaxFactory::makeBlankCollectionSyntax(SyntaxKind Kind) {
@@ -147,13 +146,15 @@ RC<RawSyntax> SyntaxFactory::createRaw(SyntaxKind Kind,
 %   end
     if (I != Elements.size())
       return nullptr;
-    return RawSyntax::make(Kind, Layout, SourcePresence::Present, Arena);
+    return RawSyntax::makeAndCalcLength(Kind, Layout, SourcePresence::Present, 
+                                        Arena);
 % elif node.is_syntax_collection():
     for (auto &E : Elements) {
       if (!canServeAsCollectionMemberRaw(SyntaxKind::${node.syntax_kind}, E))
         return nullptr;
     }
-    return RawSyntax::make(Kind, Elements, SourcePresence::Present, Arena);
+    return RawSyntax::makeAndCalcLength(Kind, Elements, SourcePresence::Present, 
+                                        Arena);
 % else:
     return nullptr;
 % end
@@ -173,7 +174,7 @@ Optional<Syntax> SyntaxFactory::createSyntax(SyntaxKind Kind,
     Layout.emplace_back(E.getRaw());
 
   if (auto Raw = createRaw(Kind, Layout, Arena))
-    return make<Syntax>(Raw);
+    return makeRoot<Syntax>(Raw);
   else
     return None;
 }
@@ -190,7 +191,7 @@ Optional<Syntax> SyntaxFactory::createSyntax(SyntaxKind Kind,
 ${node.name}
 SyntaxFactory::make${node.syntax_kind}(${child_params},
                                        RC<SyntaxArena> Arena) {
-  auto Raw = RawSyntax::make(SyntaxKind::${node.syntax_kind}, {
+  auto Raw = RawSyntax::makeAndCalcLength(SyntaxKind::${node.syntax_kind}, {
 %     for child in node.children:
 %       if child.is_optional:
     ${child.name}.hasValue() ? ${child.name}->getRaw() : nullptr,
@@ -199,7 +200,7 @@ SyntaxFactory::make${node.syntax_kind}(${child_params},
 %       end
 %     end
   }, SourcePresence::Present, Arena);
-  return make<${node.name}>(Raw);
+  return makeRoot<${node.name}>(Raw);
 }
 %   elif node.is_syntax_collection():
 ${node.name}
@@ -211,9 +212,9 @@ SyntaxFactory::make${node.syntax_kind}(
   for (auto &element : elements) {
     layout.push_back(element.getRaw());
   }
-  auto raw = RawSyntax::make(SyntaxKind::${node.syntax_kind},
+  auto raw = RawSyntax::makeAndCalcLength(SyntaxKind::${node.syntax_kind},
-                             layout, SourcePresence::Present, Arena);
+    layout, SourcePresence::Present, Arena);
-  return make<${node.name}>(raw);
+  return makeRoot<${node.name}>(raw);
 }
 %   end
 
@@ -227,8 +228,8 @@ SyntaxFactory::makeBlank${node.syntax_kind}(RC<SyntaxArena> Arena) {
     ${make_missing_child(child)},
 %       end
 %   end
-  }, SourcePresence::Present, Arena);
+  }, /*TextLength=*/0, SourcePresence::Present, Arena);
-  return make<${node.name}>(raw);
+  return makeRoot<${node.name}>(raw);
 }
 % end
 

lib/Syntax/SyntaxNodes.cpp.gyb

@@ -79,9 +79,9 @@ ${node.name} ${node.name}::add${child_elt}(${child_elt_type} ${child_elt}) {
   if (raw)
     raw = raw->append(${child_elt}.getRaw());
   else
-    raw = RawSyntax::make(SyntaxKind::${child_node.syntax_kind},
+    raw = RawSyntax::makeAndCalcLength(SyntaxKind::${child_node.syntax_kind},
-                          {${child_elt}.getRaw()}, SourcePresence::Present);
+      {${child_elt}.getRaw()}, SourcePresence::Present);
-  return Data->replaceChild<${node.name}>(raw, Cursor::${child.name});
+  return Data->replacingChild<${node.name}>(raw, Cursor::${child.name});
 }
 %     end
 
@@ -97,7 +97,7 @@ ${node.name} ${node.name}::with${child.name}(
     raw = ${make_missing_child(child)};
 % end
   }
-  return Data->replaceChild<${node.name}>(raw, Cursor::${child.name});
+  return Data->replacingChild<${node.name}>(raw, Cursor::${child.name});
 }
 
 %   end

lib/Syntax/Trivia.cpp.gyb

@@ -73,22 +73,6 @@ bool syntax::isCommentTriviaKind(TriviaKind Kind) {
   llvm_unreachable("unknown kind");
 }
 
-void TriviaPiece::accumulateAbsolutePosition(AbsolutePosition &Pos) const {
-  switch (Kind) {
-% for trivia in TRIVIAS:
-  case TriviaKind::${trivia.name}:
-% if not trivia.is_collection():
-    Pos.addText(Text.str());
-% elif trivia.is_new_line:
-    Pos.addNewlines(Count, ${trivia.characters_len()});
-% else:
-    Pos.addColumns(Count);
-% end
-  break;
-% end
-  }
-}
-
 bool TriviaPiece::trySquash(const TriviaPiece &Next) {
   if (Kind != Next.Kind) { return false; }
   switch (Kind) {

lib/SyntaxParse/RawSyntaxTokenCache.cpp

@@ -42,16 +42,14 @@ static bool shouldCacheNode(tok TokKind, size_t TextSize,
   return true;
 }
 
-RC<RawSyntax>
+RC<RawSyntax> RawSyntaxTokenCache::getToken(
-RawSyntaxTokenCache::getToken(RC<SyntaxArena> &Arena, tok TokKind,
+    RC<SyntaxArena> &Arena, tok TokKind, size_t TextLength, OwnedString Text,
-                              OwnedString Text,
+    ArrayRef<TriviaPiece> LeadingTrivia, ArrayRef<TriviaPiece> TrailingTrivia) {
-                              ArrayRef<TriviaPiece> LeadingTrivia,
-                              ArrayRef<TriviaPiece> TrailingTrivia) {
   // Determine whether this token is worth to cache.
   if (!shouldCacheNode(TokKind, Text.size(), LeadingTrivia, TrailingTrivia)) {
     // Do not use cache.
-    return RawSyntax::make(TokKind, Text, LeadingTrivia, TrailingTrivia,
+    return RawSyntax::make(TokKind, Text, TextLength, LeadingTrivia,
-                           SourcePresence::Present, Arena);
+                           TrailingTrivia, SourcePresence::Present, Arena);
   }
 
   // This node is cacheable. Get or create.
@@ -65,8 +63,8 @@ RawSyntaxTokenCache::getToken(RC<SyntaxArena> &Arena, tok TokKind,
   }
 
   // Could not found in the cache. Create it.
-  auto Raw = RawSyntax::make(TokKind, Text, LeadingTrivia, TrailingTrivia,
+  auto Raw = RawSyntax::make(TokKind, Text, TextLength, LeadingTrivia,
-                             SourcePresence::Present, Arena);
+                             TrailingTrivia, SourcePresence::Present, Arena);
   auto IDRef = ID.Intern(Arena->getAllocator());
   auto CacheNode = new (Arena) RawSyntaxCacheNode(Raw, IDRef);
   // Keep track of the created RawSyntaxCacheNode so that we can destruct it

lib/SyntaxParse/RawSyntaxTokenCache.h

@@ -62,7 +62,7 @@ class RawSyntaxTokenCache {
 
 public:
   RC<syntax::RawSyntax> getToken(RC<syntax::SyntaxArena> &Arena, tok TokKind,
-                                 OwnedString Text,
+                                 size_t TextLength, OwnedString Text,
                                  ArrayRef<syntax::TriviaPiece> LeadingTrivia,
                                  ArrayRef<syntax::TriviaPiece> TrailingTrivia);
 

lib/SyntaxParse/SyntaxTreeCreator.cpp

@@ -97,7 +97,7 @@ Optional<SourceFileSyntax>
 SyntaxTreeCreator::realizeSyntaxRoot(OpaqueSyntaxNode rootN,
                                      const SourceFile &SF) {
   auto raw = transferOpaqueNode(rootN);
-  auto rootNode = make<SourceFileSyntax>(raw);
+  auto rootNode = makeRoot<SourceFileSyntax>(raw);
 
   // Verify the tree if specified.
   if (SF.getASTContext().LangOpts.VerifySyntaxTree) {
@@ -132,8 +132,9 @@ SyntaxTreeCreator::recordToken(tok tokenKind,
                                              trailingTriviaLoc, SM, BufferID);
   StringRef tokenText = SM.extractText(tokRange, BufferID);
   auto ownedText = OwnedString::makeRefCounted(tokenText);
-  auto raw = TokenCache->getToken(Arena, tokenKind, ownedText,
+  auto raw = TokenCache->getToken(Arena, tokenKind, range.getByteLength(),
-                    syntaxLeadingTrivia.Pieces, syntaxTrailingTrivia.Pieces);
+                                  ownedText, syntaxLeadingTrivia.Pieces,
+                                  syntaxTrailingTrivia.Pieces);
   OpaqueSyntaxNode opaqueN = raw.get();
   raw.resetWithoutRelease();
   return opaqueN;
@@ -157,7 +158,9 @@ SyntaxTreeCreator::recordRawSyntax(syntax::SyntaxKind kind,
   for (OpaqueSyntaxNode opaqueN : elements) {
     parts.push_back(transferOpaqueNode(opaqueN));
   }
-  auto raw = RawSyntax::make(kind, parts, SourcePresence::Present, Arena);
+  size_t TextLength = range.isValid() ? range.getByteLength() : 0;
+  auto raw =
+      RawSyntax::make(kind, parts, TextLength, SourcePresence::Present, Arena);
   OpaqueSyntaxNode opaqueN = raw.get();
   raw.resetWithoutRelease();
   return opaqueN;

tools/swift-syntax-test/swift-syntax-test.cpp

@@ -289,26 +289,23 @@ struct ByteBasedSourceRangeSet {
   }
 };
 
-int getTokensFromFile(unsigned BufferID,
+int getTokensFromFile(
-                      LangOptions &LangOpts,
+    unsigned BufferID, LangOptions &LangOpts, SourceManager &SourceMgr,
-                      SourceManager &SourceMgr,
+    swift::DiagnosticEngine &Diags,
-                      swift::DiagnosticEngine &Diags,
+    std::vector<std::pair<RC<syntax::RawSyntax>,
-                      std::vector<std::pair<RC<syntax::RawSyntax>,
+                          syntax::AbsoluteOffsetPosition>> &Tokens) {
-                      syntax::AbsolutePosition>> &Tokens) {
   Tokens = tokenizeWithTrivia(LangOpts, SourceMgr, BufferID,
                               /*Offset=*/0, /*EndOffset=*/0,
                               &Diags);
   return EXIT_SUCCESS;
 }
 
-
+int getTokensFromFile(
-int
+    const StringRef InputFilename, LangOptions &LangOpts,
-getTokensFromFile(const StringRef InputFilename,
+    SourceManager &SourceMgr, DiagnosticEngine &Diags,
-                  LangOptions &LangOpts,
+    unsigned int &OutBufferID,
-                  SourceManager &SourceMgr,
+    std::vector<std::pair<RC<syntax::RawSyntax>,
-                  DiagnosticEngine &Diags,
+                          syntax::AbsoluteOffsetPosition>> &Tokens) {
-                  std::vector<std::pair<RC<syntax::RawSyntax>,
-                                        syntax::AbsolutePosition>> &Tokens) {
   auto Buffer = llvm::MemoryBuffer::getFile(InputFilename);
   if (!Buffer) {
     Diags.diagnose(SourceLoc(), diag::cannot_open_file,
@@ -316,8 +313,8 @@ getTokensFromFile(const StringRef InputFilename,
     return EXIT_FAILURE;
   }
 
-  auto BufferID = SourceMgr.addNewSourceBuffer(std::move(Buffer.get()));
+  OutBufferID = SourceMgr.addNewSourceBuffer(std::move(Buffer.get()));
-  return getTokensFromFile(BufferID, LangOpts, SourceMgr, Diags, Tokens);
+  return getTokensFromFile(OutBufferID, LangOpts, SourceMgr, Diags, Tokens);
 }
 
 void anchorForGetMainExecutable() {}
@@ -674,10 +671,11 @@ int doFullLexRoundTrip(const StringRef InputFilename) {
   PrintingDiagnosticConsumer DiagPrinter;
   Diags.addConsumer(DiagPrinter);
 
-  std::vector<std::pair<RC<syntax::RawSyntax>,
+  unsigned int BufferID;
-                                   syntax::AbsolutePosition>> Tokens;
+  std::vector<std::pair<RC<syntax::RawSyntax>, syntax::AbsoluteOffsetPosition>>
-  if (getTokensFromFile(InputFilename, LangOpts, SourceMgr,
+      Tokens;
-                        Diags, Tokens) == EXIT_FAILURE) {
+  if (getTokensFromFile(InputFilename, LangOpts, SourceMgr, Diags, BufferID,
+                        Tokens) == EXIT_FAILURE) {
     return EXIT_FAILURE;
   }
 
@@ -695,15 +693,20 @@ int doDumpRawTokenSyntax(const StringRef InputFile) {
   PrintingDiagnosticConsumer DiagPrinter;
   Diags.addConsumer(DiagPrinter);
 
-  std::vector<std::pair<RC<syntax::RawSyntax>,
+  unsigned int BufferID;
-                        syntax::AbsolutePosition>> Tokens;
+  std::vector<std::pair<RC<syntax::RawSyntax>, syntax::AbsoluteOffsetPosition>>
-  if (getTokensFromFile(InputFile, LangOpts, SourceMgr, Diags, Tokens) ==
+      Tokens;
-      EXIT_FAILURE) {
+  if (getTokensFromFile(InputFile, LangOpts, SourceMgr, Diags, BufferID,
+                        Tokens) == EXIT_FAILURE) {
     return EXIT_FAILURE;
   }
 
   for (auto TokAndPos : Tokens) {
-    llvm::outs() << TokAndPos.second << "\n";
+    SourceLoc Loc =
+        SourceMgr.getLocForOffset(BufferID, TokAndPos.second.getOffset());
+    unsigned Line, Column;
+    std::tie(Line, Column) = SourceMgr.getPresumedLineAndColumnForLoc(Loc);
+    llvm::outs() << Line << ":" << Column << "\n";
     TokAndPos.first->dump(llvm::outs());
     llvm::outs() << "\n";
   }
@@ -824,8 +827,7 @@ int dumpEOFSourceLoc(const char *MainExecutablePath,
 
     // To ensure the correctness of position when translated to line & column
     // pair.
-    if (SourceMgr.getPresumedLineAndColumnForLoc(EndLoc) !=
+    if (SourceMgr.getLocOffsetInBuffer(EndLoc, BufferId) != AbPos.getOffset()) {
-        AbPos.getLineAndColumn()) {
       llvm::outs() << "locations should be identical";
       return EXIT_FAILURE;
     }

unittests/Syntax/AbsolutePositionTests.cpp

@@ -1,37 +0,0 @@
-#include "swift/Syntax/SyntaxFactory.h"
-#include "swift/Syntax/SyntaxNodes.h"
-#include "swift/Syntax/SyntaxBuilders.h"
-#include "llvm/ADT/SmallString.h"
-#include "gtest/gtest.h"
-
-using namespace swift;
-using namespace swift::syntax;
-
-TEST(PositionTests, AbsolutePosition1) {
-  Trivia Leading;
-  Leading.Pieces = {TriviaPiece::newlines(2), TriviaPiece::carriageReturns(2),
-    TriviaPiece::carriageReturnLineFeeds(2)};
-  auto Token = SyntaxFactory::makeIdentifier("aaa", Leading, {});
-  AbsolutePosition Pos = Token.getAbsolutePosition();
-  ASSERT_EQ(7u, Pos.getLine());
-  ASSERT_EQ(1u, Pos.getColumn());
-  ASSERT_EQ(8u, Pos.getOffset());
-  AbsolutePosition EndPos = Token.getAbsoluteEndPositionAfterTrailingTrivia();
-  ASSERT_EQ(7u, EndPos.getLine());
-  ASSERT_EQ(4u, EndPos.getColumn());
-  ASSERT_EQ(11u, EndPos.getOffset());
-}
-
-TEST(PositionTests, AbsolutePosition2) {
-  Trivia Leading;
-  Leading.Pieces = { TriviaPiece::blockComment("/* \n\r\r\n */") };
-  auto Token = SyntaxFactory::makeIdentifier("aaa", Leading, {});
-  AbsolutePosition Pos = Token.getAbsolutePosition();
-  ASSERT_EQ(4u, Pos.getLine());
-  ASSERT_EQ(4u, Pos.getColumn());
-  ASSERT_EQ(10u, Pos.getOffset());
-  AbsolutePosition EndPos = Token.getAbsoluteEndPositionAfterTrailingTrivia();
-  ASSERT_EQ(4u, EndPos.getLine());
-  ASSERT_EQ(7u, EndPos.getColumn());
-  ASSERT_EQ(13u, EndPos.getOffset());
-}

unittests/Syntax/CMakeLists.txt

@@ -2,14 +2,12 @@ add_swift_unittest(SwiftSyntaxTests
   DeclSyntaxTests.cpp
   ExprSyntaxTests.cpp
   GenericSyntaxTests.cpp
-  RawSyntaxTests.cpp
   StmtSyntaxTests.cpp
   SyntaxCollectionTests.cpp
   ThreadSafeCachingTests.cpp
   TriviaTests.cpp
   TypeSyntaxTests.cpp
   UnknownSyntaxTests.cpp
-  AbsolutePositionTests.cpp
   )
 
 target_link_libraries(SwiftSyntaxTests

unittests/Syntax/RawSyntaxTests.cpp

@@ -1,40 +0,0 @@
-#include "swift/Parse/Token.h"
-#include "swift/Syntax/RawSyntax.h"
-#include "swift/Syntax/SyntaxFactory.h"
-#include "llvm/ADT/SmallString.h"
-#include "gtest/gtest.h"
-
-using namespace swift;
-using namespace swift::syntax;
-
-// TODO
-
-TEST(RawSyntaxTests, accumulateAbsolutePosition1) {
-  auto Token = RawSyntax::make(tok::identifier,
-                               OwnedString("aaa"),
-                               {
-                                 TriviaPiece::newlines(2),
-                                 TriviaPiece::carriageReturns(2),
-                                 TriviaPiece::carriageReturnLineFeeds(2)
-                               },
-                               {  },
-                               SourcePresence::Present);
-  AbsolutePosition Pos;
-  Token->accumulateAbsolutePosition(Pos);
-  ASSERT_EQ(7u, Pos.getLine());
-  ASSERT_EQ(4u, Pos.getColumn());
-  ASSERT_EQ(11u, Pos.getOffset());
-}
-
-TEST(RawSyntaxTests, accumulateAbsolutePosition2) {
-  auto Token = RawSyntax::make(tok::identifier,
-                               OwnedString("aaa"),
-                               {TriviaPiece::blockComment("/* \n\r\r\n */")},
-                               {  },
-                               SourcePresence::Present);
-  AbsolutePosition Pos;
-  Token->accumulateAbsolutePosition(Pos);
-  ASSERT_EQ(4u, Pos.getLine());
-  ASSERT_EQ(7u, Pos.getColumn());
-  ASSERT_EQ(13u, Pos.getOffset());
-}