Merge pull request #18016 from ahoppen/003-incremental-syntax-coloring

[libSyntax] Incremental syntax colouring

include/swift/Basic/JSONSerialization.h

@@ -27,6 +27,7 @@
 #include "llvm/Support/Regex.h"
 #include "llvm/Support/raw_ostream.h"
 
+#include <map>
 #include <vector>
 
 namespace swift {
@@ -341,6 +342,10 @@ struct unvalidatedObjectTraits : public std::integral_constant<bool,
 && !has_ObjectValidateTraits<T>::value> {};
 
 class Output {
+public:
+  using UserInfoMap = std::map<void *, void *>;
+
+private:
   enum State {
     ArrayFirstValue,
     ArrayOtherValue,
@@ -353,13 +358,19 @@ class Output {
   bool PrettyPrint;
   bool NeedBitValueComma;
   bool EnumerationMatchFound;
+  UserInfoMap UserInfo;
 
 public:
-  Output(llvm::raw_ostream &os, bool PrettyPrint = true) : Stream(os),
-      PrettyPrint(PrettyPrint), NeedBitValueComma(false),
-      EnumerationMatchFound(false) {}
+  Output(llvm::raw_ostream &os, UserInfoMap UserInfo = {},
+         bool PrettyPrint = true)
+      : Stream(os), PrettyPrint(PrettyPrint), NeedBitValueComma(false),
+        EnumerationMatchFound(false), UserInfo(UserInfo) {}
   virtual ~Output() = default;
 
+  UserInfoMap &getUserInfo() {
+    return UserInfo;
+  }
+
   unsigned beginArray();
   bool preflightElement(unsigned, void *&);
   void postflightElement(void*);

include/swift/Basic/OwnedString.h

@@ -71,7 +71,7 @@ public:
   OwnedString(): OwnedString(nullptr, 0, StringOwnership::Unowned) {}
 
   OwnedString(const char *Data, size_t Length):
-    OwnedString(Data, Length, StringOwnership::Unowned) {}
+    OwnedString(Data, Length, StringOwnership::Copied) {}
 
   OwnedString(StringRef Str) : OwnedString(Str.data(), Str.size()) {}
 
@@ -106,7 +106,7 @@ public:
     return *this;
   }
 
-  OwnedString copy() {
+  OwnedString copy() const {
     return OwnedString(Data, Length, StringOwnership::Copied);
   }
 

include/swift/Parse/SyntaxParsingCache.h

@@ -47,6 +47,13 @@ namespace swift {
 
 using namespace swift::syntax;
 
+struct SyntaxReuseRegion {
+  /// The byte offset at which the range begins
+  uintptr_t Start;
+  /// The byte offset at which the end ends
+  uintptr_t End;
+};
+
 class SyntaxParsingCache {
   /// The syntax tree prior to the edit
   SourceFileSyntax OldSyntaxTree;
@@ -61,7 +68,7 @@ class SyntaxParsingCache {
 
   /// If \c RecordReuseInformation buffer offsets of ranges that have been
   /// successfully looked up in this cache are stored.
-  std::vector<std::pair<unsigned, unsigned>> ReusedRanges;
+  std::vector<SyntaxReuseRegion> ReusedRanges;
 
 public:
   SyntaxParsingCache(SourceFileSyntax OldSyntaxTree)
@@ -85,7 +92,7 @@ public:
   /// Return the ranges of the new source file that have been successfully
   /// looked up in this cache as a (start, end) pair of byte offsets in the
   /// post-edit file.
-  std::vector<std::pair<unsigned, unsigned>> getReusedRanges() const {
+  std::vector<SyntaxReuseRegion> getReusedRanges() const {
     return ReusedRanges;
   }
 

include/swift/Subsystems.h

@@ -61,6 +61,7 @@ namespace swift {
   class SILParserTUState;
   class SourceFile;
   class SourceManager;
+  class SyntaxParsingCache;
   class Token;
   class TopLevelContext;
   struct TypeLoc;
@@ -327,7 +328,8 @@ namespace swift {
   class ParserUnit {
   public:
     ParserUnit(SourceManager &SM, unsigned BufferID,
-               const LangOptions &LangOpts, StringRef ModuleName);
+               const LangOptions &LangOpts, StringRef ModuleName,
+               SyntaxParsingCache *SyntaxCache = nullptr);
     ParserUnit(SourceManager &SM, unsigned BufferID);
     ParserUnit(SourceManager &SM, unsigned BufferID,
                unsigned Offset, unsigned EndOffset);

include/swift/Syntax/Serialization/SyntaxSerialization.h

@@ -27,6 +27,10 @@
 namespace swift {
 namespace json {
 
+/// The associated value will be interpreted as \c bool. If \c true the node IDs
+/// will not be included in the serialized JSON.
+static void *DontSerializeNodeIdsUserInfoKey = &DontSerializeNodeIdsUserInfoKey;
+
 /// Serialization traits for SourcePresence.
 template <>
 struct ScalarEnumerationTraits<syntax::SourcePresence> {
@@ -141,9 +145,13 @@ struct ObjectTraits<syntax::RawSyntax> {
     }
     auto presence = value.getPresence();
     out.mapRequired("presence", presence);
+
+    bool omitNodeId = (bool)out.getUserInfo()[DontSerializeNodeIdsUserInfoKey];
+    if (!omitNodeId) {
       auto nodeId = value.getId();
       out.mapRequired("id", nodeId);
     }
+  }
 };
 
 template<>

lib/FrontendTool/FrontendTool.cpp

@@ -242,7 +242,8 @@ static bool emitLoadedModuleTraceIfNeeded(ASTContext &ctxt,
   std::string stringBuffer;
   {
     llvm::raw_string_ostream memoryBuffer(stringBuffer);
-    json::Output jsonOutput(memoryBuffer, /*PrettyPrint=*/false);
+    json::Output jsonOutput(memoryBuffer, /*UserInfo=*/{},
+                            /*PrettyPrint=*/false);
     json::jsonize(jsonOutput, trace, /*Required=*/true);
   }
   stringBuffer += "\n";
@@ -289,7 +290,7 @@ static bool emitSyntax(SourceFile *SF, LangOptions &LangOpts,
   auto os = getFileOutputStream(OutputFilename, SF->getASTContext());
   if (!os) return true;
 
-  json::Output jsonOut(*os, /*PrettyPrint=*/false);
+  json::Output jsonOut(*os, /*UserInfo=*/{}, /*PrettyPrint=*/false);
   auto Root = SF->getSyntaxRoot().getRaw();
   jsonOut << *Root;
   *os << "\n";

lib/Parse/Parser.cpp

@@ -1018,9 +1018,11 @@ ParserUnit::ParserUnit(SourceManager &SM, unsigned BufferID)
 }
 
 ParserUnit::ParserUnit(SourceManager &SM, unsigned BufferID,
-                       const LangOptions &LangOpts, StringRef ModuleName)
+                       const LangOptions &LangOpts, StringRef ModuleName,
+                       SyntaxParsingCache *SyntaxCache)
     : Impl(*new Implementation(SM, BufferID, LangOpts, ModuleName)) {
 
+  Impl.SF->SyntaxParsingCache = SyntaxCache;
   Impl.TheParser.reset(new Parser(BufferID, *Impl.SF, nullptr));
 }
 

test/incrParse/invalid.swift

@@ -1,8 +1,17 @@
 // RUN: %empty-directory(%t)
 // RUN: %incparse-test %s --test-case NO_CHANGES
+// RUN: %incparse-test %s --test-case NESTED_INITIALIZERS
 
 func start() {}
 
 class Bar
 
 let y = 1
+
+class NestedInitializers {
+  <<NESTED_INITIALIZERS<|||init() {>>>
+    init() {
+
+    }
+  <<NESTED_INITIALIZERS<|||}>>>
+}

tools/SourceKit/include/SourceKit/Core/LangSupport.h

@@ -188,6 +188,13 @@ struct DiagnosticEntryInfo : DiagnosticEntryInfoBase {
   SmallVector<DiagnosticEntryInfoBase, 1> Notes;
 };
 
+struct SourceFileRange {
+  /// The byte offset at which the range begins
+  uintptr_t Start;
+  /// The byte offset at which the end ends
+  uintptr_t End;
+};
+
 class EditorConsumer {
   virtual void anchor();
 public:
@@ -240,6 +247,10 @@ public:
   virtual bool handleSerializedSyntaxTree(StringRef Text) = 0;
   virtual bool syntaxTreeEnabled() = 0;
 
+  virtual bool syntaxReuseInfoEnabled() = 0;
+  virtual bool handleSyntaxReuseRegions(
+      std::vector<SourceFileRange> ReuseRegions) = 0;
+
   virtual void finished() {}
 
   // FIXME: This is just for bootstrapping incremental syntax tree parsing.

tools/SourceKit/lib/SwiftLang/SwiftEditor.cpp

@@ -41,6 +41,7 @@
 #include "swift/Syntax/SyntaxNodes.h"
 
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/FileSystem.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Mutex.h"
 
@@ -773,6 +774,9 @@ class SwiftDocumentSyntaxInfo {
   unsigned BufferID;
   std::vector<std::string> Args;
   std::string PrimaryFile;
+  /// Whether or not the AST stored in the source file is up-to-date or just an
+  /// artifact of incremental syntax parsing
+  bool HasUpToDateAST;
 
 public:
   SwiftDocumentSyntaxInfo(const CompilerInvocation &CompInv,
@@ -792,10 +796,15 @@ public:
     Parser.reset(
       new ParserUnit(SM, BufferID,
                      CompInv.getLangOptions(),
-                     CompInv.getModuleName())
+                     CompInv.getModuleName(),
+                     CompInv.getMainFileSyntaxParsingCache())
     );
 
     Parser->getDiagnosticEngine().addConsumer(DiagConsumer);
+
+    // If there is a syntax parsing cache, incremental syntax parsing is
+    // performed and thus the generated AST may not be up-to-date.
+    HasUpToDateAST = CompInv.getMainFileSyntaxParsingCache() == nullptr;
   }
 
   void parse() {
@@ -824,6 +833,8 @@ public:
     return SM;
   }
 
+  bool hasUpToDateAST() { return HasUpToDateAST; }
+
   ArrayRef<DiagnosticEntryInfo> getDiagnostics() {
     return DiagConsumer.getDiagnosticsForBuffer(BufferID);
   }
@@ -1138,6 +1149,8 @@ struct SwiftEditorDocument::Implementation {
   llvm::Optional<SwiftEditorCharRange> AffectedRange;
   /// Whether the last operation was an edit rather than a document open
   bool Edited;
+  /// The syntax tree of the document
+  llvm::Optional<SourceFileSyntax> SyntaxTree;
 
   std::vector<DiagnosticEntryInfo> ParserDiagnostics;
   RefPtr<SwiftDocumentSemanticInfo> SemanticInfo;
@@ -1853,7 +1866,8 @@ void SwiftEditorDocument::updateSemaInfo() {
 }
 
 void SwiftEditorDocument::parse(ImmutableTextSnapshotRef Snapshot,
-                                SwiftLangSupport &Lang, bool BuildSyntexTree) {
+                                SwiftLangSupport &Lang, bool BuildSyntaxTree,
+                                SyntaxParsingCache *SyntaxCache) {
   llvm::sys::ScopedLock L(Impl.AccessMtx);
 
   assert(Impl.SemanticInfo && "Impl.SemanticInfo must be set");
@@ -1876,7 +1890,11 @@ void SwiftEditorDocument::parse(ImmutableTextSnapshotRef Snapshot,
     Lang.getASTManager().
       initCompilerInvocation(CompInv, Args, StringRef(), Error);
   }
-  CompInv.getLangOptions().BuildSyntaxTree = BuildSyntexTree;
+  CompInv.getLangOptions().BuildSyntaxTree = BuildSyntaxTree;
+  CompInv.setMainFileSyntaxParsingCache(SyntaxCache);
+  // When reuse parts of the syntax tree from a SyntaxParsingCache, not
+  // all tokens are visited and thus token collection is invalid
+  CompInv.getLangOptions().CollectParsedToken = (SyntaxCache == nullptr);
   // Access to Impl.SyntaxInfo is guarded by Impl.AccessMtx
   Impl.SyntaxInfo.reset(
     new SwiftDocumentSyntaxInfo(CompInv, Snapshot, Args, Impl.FilePath));
@@ -1892,7 +1910,7 @@ void SwiftEditorDocument::readSyntaxInfo(EditorConsumer &Consumer) {
   if (Consumer.syntaxTreeEnabled()) {
     std::string SyntaxContent;
     llvm::raw_string_ostream OS(SyntaxContent);
-    json::Output JsonOut(OS, /*PrettyPrint=*/false);
+    json::Output JsonOut(OS, /*UserInfo=*/{}, /*PrettyPrint=*/false);
     auto Root = Impl.SyntaxInfo->getSourceFile().getSyntaxRoot().getRaw();
     JsonOut << *Root;
     Consumer.handleSerializedSyntaxTree(OS.str());
@@ -1907,6 +1925,7 @@ void SwiftEditorDocument::readSyntaxInfo(EditorConsumer &Consumer) {
     auto Classification = Classifier.classify(SyntaxTree);
     SyntaxToSyntaxMapConverter Printer(NewMap, Classification);
     Printer.writeToSyntaxMap(SyntaxTree);
+    Impl.SyntaxTree.emplace(SyntaxTree);
   } else {
     ide::SyntaxModelContext ModelContext(Impl.SyntaxInfo->getSourceFile());
 
@@ -1989,6 +2008,29 @@ const CodeFormatOptions &SwiftEditorDocument::getFormatOptions() {
   return Impl.FormatOptions;
 }
 
+const llvm::Optional<swift::SourceFileSyntax> &
+SwiftEditorDocument::getSyntaxTree() const {
+  return Impl.SyntaxTree;
+}
+
+const SourceManager &SwiftEditorDocument::getSourceManager() const {
+  return Impl.SyntaxInfo->getSourceManager();
+}
+
+SourceManager &SwiftEditorDocument::getSourceManager() {
+  return Impl.SyntaxInfo->getSourceManager();
+}
+
+unsigned SwiftEditorDocument::getBufferID() const {
+  return Impl.SyntaxInfo->getBufferID();
+}
+
+std::string SwiftEditorDocument::getFilePath() const { return Impl.FilePath; }
+
+bool SwiftEditorDocument::hasUpToDateAST() const {
+  return Impl.SyntaxInfo->hasUpToDateAST();
+}
+
 void SwiftEditorDocument::formatText(unsigned Line, unsigned Length,
                                      EditorConsumer &Consumer) {
   auto SyntaxInfo = Impl.getSyntaxInfo();
@@ -2212,10 +2254,98 @@ void SwiftLangSupport::editorClose(StringRef Name, bool RemoveCache) {
 // EditorReplaceText
 //===----------------------------------------------------------------------===//
 
+void verifyIncrementalParse(SwiftEditorDocumentRef EditorDoc,
+                            unsigned EditOffset, unsigned EditLength,
+                            StringRef PreEditText, StringRef ReplaceText) {
+  swift::json::Output::UserInfoMap JsonUserInfo;
+  JsonUserInfo[swift::json::DontSerializeNodeIdsUserInfoKey] =
+      reinterpret_cast<void *>(true);
+
+  // Dump the incremental syntax tree
+  std::string IncrTreeString;
+  llvm::raw_string_ostream IncrTreeStream(IncrTreeString);
+  swift::json::Output IncrTreeOutput(IncrTreeStream, JsonUserInfo);
+  IncrTreeOutput << *EditorDoc->getSyntaxTree()->getRaw();
+
+  // Reparse the file from scratch
+  CompilerInvocation Invocation;
+  Invocation.getLangOptions().BuildSyntaxTree = true;
+  std::vector<std::string> Args;
+  SwiftDocumentSyntaxInfo ScratchSyntaxInfo(Invocation,
+                                            EditorDoc->getLatestSnapshot(),
+                                            Args, EditorDoc->getFilePath());
+  ScratchSyntaxInfo.parse();
+
+  // Dump the from-scratch syntax tree
+  std::string FromScratchTreeString;
+  llvm::raw_string_ostream ScratchTreeStream(FromScratchTreeString);
+  swift::json::Output ScratchTreeOutput(ScratchTreeStream, JsonUserInfo);
+  auto SyntaxRoot = ScratchSyntaxInfo.getSourceFile().getSyntaxRoot();
+  ScratchTreeOutput << *SyntaxRoot.getRaw();
+
+  // If the serialized format of the two trees doesn't match incremental parsing
+  // we have found an error.
+  if (IncrTreeStream.str().compare(ScratchTreeStream.str())) {
+    LOG_SECTION("Incremental Parsing", Warning) {
+      Log->getOS() << "Incremental parsing different to from scratch parsing\n";
+      Log->getOS() << "Edit was " << EditOffset << "-"
+                   << (EditOffset + EditLength) << "='" << ReplaceText << "'"
+                   << " pre-edit-text: '" << PreEditText << "'\n";
+
+      SmallString<32> DirectoryName;
+      if (llvm::sys::fs::createUniqueDirectory(
+              "SourceKit-IncrementalParsing-Inconsistency", DirectoryName)) {
+        Log->getOS() << "Failed to create log directory\n";
+      }
+
+      std::error_code ErrorCode;
+
+      // Write the incremental syntax tree
+      auto IncrTreeFilename = DirectoryName + "/incrementalTree.json";
+      llvm::raw_fd_ostream IncrementalFilestream(
+          IncrTreeFilename.str(), ErrorCode, llvm::sys::fs::F_RW);
+      IncrementalFilestream << IncrTreeStream.str();
+      if (ErrorCode) {
+        Log->getOS() << "Failed to write incremental syntax tree to "
+                     << IncrTreeFilename << "(error code " << ErrorCode.value()
+                     << ": " << ErrorCode.message() << ")\n";
+      } else {
+        Log->getOS() << "Incremental syntax tree written to "
+                     << IncrTreeFilename << '\n';
+      }
+
+      // Write from-scratch syntax tree
+      auto ScratchTreeFilename = DirectoryName + "/fromScratchTree.json";
+      llvm::raw_fd_ostream ScratchTreeFilestream(
+          ScratchTreeFilename.str(), ErrorCode, llvm::sys::fs::F_RW);
+      ScratchTreeFilestream << ScratchTreeStream.str();
+      if (ErrorCode) {
+        Log->getOS() << "Failed to write from-scratch syntax tree to "
+                     << ScratchTreeFilename << "(error code "
+                     << ErrorCode.value() << ": " << ErrorCode.message()
+                     << ")\n";
+      } else {
+        Log->getOS() << "From-scratch syntax tree written to "
+                     << ScratchTreeFilename << '\n';
+      }
+
+      // Write source file
+      auto SourceFilename = DirectoryName + "/postEditSource.swift";
+      llvm::raw_fd_ostream SourceFilestream(SourceFilename.str(), ErrorCode,
+                                            llvm::sys::fs::F_RW);
+      auto FileBuffer = EditorDoc->getLatestSnapshot()->getBuffer();
+      SourceFilestream << FileBuffer->getText();
+    }
+  }
+}
+
 void SwiftLangSupport::editorReplaceText(StringRef Name,
                                          llvm::MemoryBuffer *Buf,
                                          unsigned Offset, unsigned Length,
                                          EditorConsumer &Consumer) {
+  bool LogReuseRegions = ::getenv("SOURCEKIT_LOG_INCREMENTAL_REUSE_REGIONS");
+  bool ValidateSyntaxTree = ::getenv("SOURCEKIT_INCREMENTAL_PARSE_VALIDATION");
+
   auto EditorDoc = EditorDocuments.getByUnresolvedName(Name);
   if (!EditorDoc) {
     Consumer.handleRequestError("No associated Editor Document");
@@ -2224,13 +2354,80 @@ void SwiftLangSupport::editorReplaceText(StringRef Name,
 
   ImmutableTextSnapshotRef Snapshot;
   if (Length != 0 || Buf->getBufferSize() != 0) {
+    std::string PreEditText;
+    if (ValidateSyntaxTree) {
+      auto CurBuffer = EditorDoc->getLatestSnapshot()->getBuffer();
+      auto BufferStart = CurBuffer->getInternalBuffer()->getBufferStart();
+      StringRef PreEditTextRef(BufferStart + Offset, Length);
+      PreEditText = PreEditTextRef.str();
+    }
     Snapshot = EditorDoc->replaceText(Offset, Length, Buf,
                                       Consumer.needsSemanticInfo());
     assert(Snapshot);
     bool BuildSyntaxTree = Consumer.syntaxTreeEnabled() ||
                            Consumer.forceLibSyntaxBasedProcessing();
-    EditorDoc->parse(Snapshot, *this, BuildSyntaxTree);
+
+    llvm::Optional<SyntaxParsingCache> SyntaxCache;
+    if (EditorDoc->getSyntaxTree().hasValue()) {
+      SyntaxCache.emplace(EditorDoc->getSyntaxTree().getValue());
+      SyntaxCache->addEdit(Offset, Offset + Length, Buf->getBufferSize());
+      SyntaxCache->setRecordReuseInformation();
+    }
+
+    SyntaxParsingCache *SyntaxCachePtr = nullptr;
+    if (SyntaxCache.hasValue()) {
+      SyntaxCachePtr = SyntaxCache.getPointer();
+    }
+    EditorDoc->parse(Snapshot, *this, BuildSyntaxTree, SyntaxCachePtr);
     EditorDoc->readSyntaxInfo(Consumer);
+
+    // Log reuse information
+    if (SyntaxCache.hasValue()) {
+      // Avoid computing the reused ranges if the consumer doesn't care about
+      // them
+      if (Consumer.syntaxReuseInfoEnabled()) {
+        auto ReuseRegions = SyntaxCache->getReusedRanges();
+        std::vector<SourceFileRange> ReuseRegionOffsets;
+        ReuseRegionOffsets.reserve(ReuseRegions.size());
+        for (auto ReuseRegion : ReuseRegions) {
+          auto Start = ReuseRegion.Start;
+          auto End = ReuseRegion.End;
+          ReuseRegionOffsets.push_back({Start, End});
+        }
+        Consumer.handleSyntaxReuseRegions(ReuseRegionOffsets);
+      }
+      if (LogReuseRegions) {
+        LOG_SECTION("SyntaxCache", InfoHighPrio) {
+          Log->getOS() << "Reused ";
+
+          bool FirstIteration = true;
+          unsigned LastPrintedBufferID;
+          for (auto ReuseRegion : SyntaxCache->getReusedRanges()) {
+            if (!FirstIteration) {
+              Log->getOS() << ", ";
+            } else {
+              FirstIteration = false;
+            }
+
+            const SourceManager &SM = EditorDoc->getSourceManager();
+            unsigned BufferID = EditorDoc->getBufferID();
+            auto Start = SM.getLocForOffset(BufferID, ReuseRegion.Start);
+            auto End = SM.getLocForOffset(BufferID, ReuseRegion.End);
+
+            Start.print(Log->getOS(), SM, LastPrintedBufferID);
+            Log->getOS() << " - ";
+            End.print(Log->getOS(), SM, LastPrintedBufferID);
+          }
+        }
+      }
+    } else {
+      Consumer.handleSyntaxReuseRegions({});
+    }
+
+    if (ValidateSyntaxTree) {
+      verifyIncrementalParse(EditorDoc, Offset, Length, PreEditText,
+                             Buf->getBuffer());
+    }
   } else {
     Snapshot = EditorDoc->getLatestSnapshot();
   }
@@ -2258,6 +2455,13 @@ void SwiftLangSupport::editorFormatText(StringRef Name, unsigned Line,
     return;
   }
 
+  if (!EditorDoc->hasUpToDateAST()) {
+    // An up-to-date AST is needed for formatting. If it does not exist, fall
+    // back to a full reparse of the file
+    EditorDoc->parse(EditorDoc->getLatestSnapshot(), *this,
+                     /*BuildSyntaxTree=*/true);
+  }
+
   EditorDoc->formatText(Line, Length, Consumer);
 }
 

tools/SourceKit/lib/SwiftLang/SwiftLangSupport.h

@@ -93,7 +93,8 @@ public:
   ImmutableTextSnapshotRef getLatestSnapshot() const;
 
   void parse(ImmutableTextSnapshotRef Snapshot, SwiftLangSupport &Lang,
-             bool BuildSyntaxTree);
+             bool BuildSyntaxTree,
+             swift::SyntaxParsingCache *SyntaxCache = nullptr);
   void readSyntaxInfo(EditorConsumer &consumer);
   void readSemanticInfo(ImmutableTextSnapshotRef Snapshot,
                         EditorConsumer& Consumer);
@@ -106,6 +107,20 @@ public:
 
   static void reportDocumentStructure(swift::SourceFile &SrcFile,
                                       EditorConsumer &Consumer);
+
+  const llvm::Optional<swift::SourceFileSyntax> &getSyntaxTree() const;
+
+  const swift::SourceManager &getSourceManager() const;
+  swift::SourceManager &getSourceManager();
+
+  /// Get the buffer ID of this file in its source manager
+  unsigned getBufferID() const;
+
+  std::string getFilePath() const;
+
+  /// Whether or not the AST stored for this document is up-to-date or just an
+  /// artifact of incremental syntax parsing
+  bool hasUpToDateAST() const;
 };
 
 typedef IntrusiveRefCntPtr<SwiftEditorDocument> SwiftEditorDocumentRef;

tools/SourceKit/tools/sourcekitd/lib/API/Requests.cpp

@@ -75,6 +75,7 @@ struct SKEditorConsumerOptions {
   bool EnableDiagnostics = false;
   bool EnableSyntaxTree = false;
   bool SyntacticOnly = false;
+  bool EnableSyntaxReuseInfo = false;
   // FIXME: This is just for bootstrapping incremental syntax tree parsing.
   // Remove it once when we are able to incrementally transfer the syntax tree
   bool ForceLibSyntaxBasedProcessing = false;
@@ -441,6 +442,9 @@ void handleRequestImpl(sourcekitd_object_t ReqObj, ResponseReceiver Rec) {
     int64_t ForceLibSyntaxBasedProcessing = false;
     Req.getInt64(KeyForceLibSyntaxBasedProcessing,
                  ForceLibSyntaxBasedProcessing, /*isOptional=*/true);
+    int64_t EnableSyntaxReuseInfo = false;
+    Req.getInt64(KeyEnableSyntaxReuseRegions, EnableSyntaxReuseInfo,
+                 /*isOptional=*/true);
 
     SKEditorConsumerOptions Opts;
     Opts.EnableSyntaxMap = EnableSyntaxMap;
@@ -448,6 +452,7 @@ void handleRequestImpl(sourcekitd_object_t ReqObj, ResponseReceiver Rec) {
     Opts.EnableDiagnostics = EnableDiagnostics;
     Opts.EnableSyntaxTree = EnableSyntaxTree;
     Opts.SyntacticOnly = SyntacticOnly;
+    Opts.EnableSyntaxReuseInfo = EnableSyntaxReuseInfo;
     Opts.ForceLibSyntaxBasedProcessing = ForceLibSyntaxBasedProcessing;
     return Rec(editorOpen(*Name, InputBuf.get(), Opts, Args));
   }
@@ -487,6 +492,9 @@ void handleRequestImpl(sourcekitd_object_t ReqObj, ResponseReceiver Rec) {
     Req.getInt64(KeyForceLibSyntaxBasedProcessing,
                  ForceLibSyntaxBasedProcessing,
                  /*isOptional=*/true);
+    int64_t EnableSyntaxReuseInfo = false;
+    Req.getInt64(KeyEnableSyntaxReuseRegions, EnableSyntaxReuseInfo,
+                 /*isOptional=*/true);
 
     SKEditorConsumerOptions Opts;
     Opts.EnableSyntaxMap = EnableSyntaxMap;
@@ -494,6 +502,7 @@ void handleRequestImpl(sourcekitd_object_t ReqObj, ResponseReceiver Rec) {
     Opts.EnableDiagnostics = EnableDiagnostics;
     Opts.EnableSyntaxTree = EnableSyntaxTree;
     Opts.SyntacticOnly = SyntacticOnly;
+    Opts.EnableSyntaxReuseInfo = EnableSyntaxReuseInfo;
     Opts.ForceLibSyntaxBasedProcessing = ForceLibSyntaxBasedProcessing;
 
     return Rec(editorReplaceText(*Name, InputBuf.get(), Offset, Length, Opts));
@@ -2072,6 +2081,11 @@ public:
   bool handleSourceText(StringRef Text) override;
   bool handleSerializedSyntaxTree(StringRef Text) override;
   bool syntaxTreeEnabled() override { return Opts.EnableSyntaxTree; }
+
+  bool syntaxReuseInfoEnabled() override { return Opts.EnableSyntaxReuseInfo; }
+  bool handleSyntaxReuseRegions(
+      std::vector<SourceFileRange> ReuseRegions) override;
+
   void finished() override {
     if (RespReceiver)
       RespReceiver(createResponse());
@@ -2085,8 +2099,8 @@ public:
 } // end anonymous namespace
 
 static sourcekitd_response_t
-editorOpen(StringRef Name, llvm::MemoryBuffer *Buf, SKEditorConsumerOptions Opts,
-           ArrayRef<const char *> Args) {
+editorOpen(StringRef Name, llvm::MemoryBuffer *Buf,
+           SKEditorConsumerOptions Opts, ArrayRef<const char *> Args) {
   SKEditorConsumer EditC(Opts);
   LangSupport &Lang = getGlobalContext().getSwiftLangSupport();
   Lang.editorOpen(Name, Buf, EditC, Args);
@@ -2417,6 +2431,20 @@ bool SKEditorConsumer::handleSerializedSyntaxTree(StringRef Text) {
   return true;
 }
 
+bool SKEditorConsumer::handleSyntaxReuseRegions(
+    std::vector<SourceFileRange> ReuseRegions) {
+  if (Opts.EnableSyntaxReuseInfo) {
+    auto Array = Dict.setArray(KeySyntaxReuseRegions);
+
+    for (auto Region : ReuseRegions) {
+      auto SubDict = Array.appendDictionary();
+      SubDict.set(KeyOffset, Region.Start);
+      SubDict.set(KeyLength, Region.End - Region.Start);
+    }
+  }
+  return true;
+}
+
 static sourcekitd_response_t
 editorFindUSR(StringRef DocumentName, StringRef USR) {
   ResponseBuilder RespBuilder;

tools/swift-syntax-test/swift-syntax-test.cpp

@@ -198,8 +198,8 @@ struct ByteBasedSourceRange {
   ByteBasedSourceRange(unsigned Start, unsigned End) : Start(Start), End(End) {
     assert(Start <= End);
   }
-  ByteBasedSourceRange(std::pair<unsigned, unsigned> Pair)
-      : ByteBasedSourceRange(Pair.first, Pair.second) {}
+  ByteBasedSourceRange(SyntaxReuseRegion Pair)
+      : ByteBasedSourceRange(Pair.Start, Pair.End) {}
   ByteBasedSourceRange() : ByteBasedSourceRange(0, 0) {}
 
   ByteBasedSourceRange intersect(const ByteBasedSourceRange &Other) {
@@ -228,8 +228,7 @@ struct ByteBasedSourceRangeSet {
 
   ByteBasedSourceRangeSet() {}
 
-  ByteBasedSourceRangeSet(
-      std::vector<std::pair<unsigned, unsigned>> PairVector) {
+  ByteBasedSourceRangeSet(std::vector<SyntaxReuseRegion> PairVector) {
     for (auto Pair : PairVector) {
       addRange(Pair);
     }
@@ -439,11 +438,11 @@ void printVisualNodeReuseInformation(SourceManager &SourceMgr,
 
   for (auto ReuseRange : Cache->getReusedRanges()) {
     // Print region that was not reused
-    PrintReparsedRegion(SourceText, CurrentOffset, ReuseRange.first);
+    PrintReparsedRegion(SourceText, CurrentOffset, ReuseRange.Start);
 
-    llvm::outs() << SourceText.substr(ReuseRange.first,
-                                      ReuseRange.second - ReuseRange.first);
-    CurrentOffset = ReuseRange.second;
+    llvm::outs() << SourceText.substr(ReuseRange.Start,
+                                      ReuseRange.End - ReuseRange.Start);
+    CurrentOffset = ReuseRange.End;
   }
   PrintReparsedRegion(SourceText, CurrentOffset, SourceText.size());
   if (useColoredOutput())
@@ -460,8 +459,8 @@ void saveReuseLog(SourceManager &SourceMgr, unsigned BufferID,
   assert(!ErrorCode && "Unable to open incremental usage log");
 
   for (auto ReuseRange : Cache->getReusedRanges()) {
-    SourceLoc Start = SourceMgr.getLocForOffset(BufferID, ReuseRange.first);
-    SourceLoc End = SourceMgr.getLocForOffset(BufferID, ReuseRange.second);
+    SourceLoc Start = SourceMgr.getLocForOffset(BufferID, ReuseRange.Start);
+    SourceLoc End = SourceMgr.getLocForOffset(BufferID, ReuseRange.End);
 
     ReuseLog << "Reused ";
     Start.printLineAndColumn(ReuseLog, SourceMgr, BufferID);

unittests/Basic/JSONSerialization.cpp

@@ -29,6 +29,8 @@ struct Root {
 namespace swift {
 namespace json {
 
+void *IncludeSpecialValueUserInfoKey = &IncludeSpecialValueUserInfoKey;
+
 template <> struct ObjectTraits<Leaf> {
   static void mapping(Output &out, Leaf &value) {
     switch (value.Val) {
@@ -71,6 +73,10 @@ template <> struct ObjectTraits<Root> {
     out.mapRequired("Children", value.Children);
     out.mapRequired("Empty", value.Empty);
     out.mapRequired("Side", value.Side);
+    if ((bool)out.getUserInfo()[IncludeSpecialValueUserInfoKey]) {
+      std::string Value = "42";
+      out.mapRequired("SpecialValue", Value);
+    }
   }
 };
 
@@ -84,7 +90,7 @@ TEST(JSONSerialization, basicCompact) {
   Root RootObj{"foo", {&LeafObj0, &LeafObj1, nullptr, &LeafObj2}, {}, nullptr};
   std::string Buffer;
   llvm::raw_string_ostream Stream(Buffer);
-  swift::json::Output Out(Stream, /*PrettyPrint=*/false);
+  swift::json::Output Out(Stream, /*UserInfo=*/{}, /*PrettyPrint=*/false);
 
   Out << RootObj;
   Stream.flush();
@@ -121,3 +127,24 @@ TEST(JSONSerialization, basicPretty) {
   "Side": null
 })""");
 }
+
+TEST(JSONSerialization, basicUserInfo) {
+  Root RootObj{"foo", {}, {}, nullptr};
+  std::string Buffer;
+  llvm::raw_string_ostream Stream(Buffer);
+
+  swift::json::Output::UserInfoMap UserInfo;
+  UserInfo[swift::json::IncludeSpecialValueUserInfoKey] = (void *)true;
+  swift::json::Output Out(Stream, UserInfo);
+
+  Out << RootObj;
+  Stream.flush();
+
+  EXPECT_EQ(Buffer, R"""({
+  "Name": "foo",
+  "Children": [],
+  "Empty": [],
+  "Side": null,
+  "SpecialValue": "42"
+})""");
+}

unittests/Basic/OwnedStringTest.cpp

@@ -159,7 +159,7 @@ TEST(OwnedStringTest, copy_constructor_original_not_copy) {
   // Make sure updating the original pointer doesn't affect the copy.
   data[0] = 'a';
 
-  EXPECT_EQ("atring", str);
+  EXPECT_EQ("string", str);
 }
 
 TEST(OwnedStringTest, copy_constructor_original_copy) {

unittests/SourceKit/SwiftLang/CursorInfoTest.cpp

@@ -92,6 +92,12 @@ class NullEditorConsumer : public EditorConsumer {
   bool syntaxTreeEnabled() override { return false; }
   bool forceLibSyntaxBasedProcessing() override { return false; }
 
+  bool syntaxReuseInfoEnabled() override { return false; }
+
+  bool handleSyntaxReuseRegions(
+      std::vector<SourceFileRange> ReuseRegions) override {
+    return false;
+  }
 public:
   bool needsSema = false;
 };

unittests/SourceKit/SwiftLang/EditingTest.cpp

@@ -100,6 +100,13 @@ private:
   bool handleSerializedSyntaxTree(StringRef Text) override { return false; }
   bool syntaxTreeEnabled() override { return false; }
   bool forceLibSyntaxBasedProcessing() override { return false; }
+
+  bool syntaxReuseInfoEnabled() override { return false; }
+
+  bool handleSyntaxReuseRegions(
+      std::vector<SourceFileRange> ReuseRegions) override {
+    return false;
+  }
 };
 
 struct DocUpdateMutexState {

utils/gyb_sourcekit_support/UIDs.py

@@ -47,8 +47,10 @@ UID_KEYS = [
     KEY('Length', 'key.length'),
     KEY('SourceFile', 'key.sourcefile'),
     KEY('SerializedSyntaxTree', 'key.serialized_syntax_tree'),
+    KEY('SyntaxReuseRegions', 'key.syntaxreuseregions'),
     KEY('SourceText', 'key.sourcetext'),
     KEY('ForceLibSyntaxBasedProcessing', 'key.forcelibsyntaxbasedprocessing'),
+    KEY('EnableSyntaxReuseRegions', 'key.enablesyntaxreuseregions'),
     KEY('EnableSyntaxMap', 'key.enablesyntaxmap'),
     KEY('EnableSyntaxTree', 'key.enablesyntaxtree'),
     KEY('EnableStructure', 'key.enablesubstructure'),