Merge remote-tracking branch 'origin/master' into master-next

include/swift/AST/Module.h

@@ -888,7 +888,8 @@ public:
   ASTStage_t ASTStage = Parsing;
 
   SourceFile(ModuleDecl &M, SourceFileKind K, Optional<unsigned> bufferID,
-             ImplicitModuleImportKind ModImpKind, bool KeepSyntaxInfo);
+             ImplicitModuleImportKind ModImpKind, bool KeepParsedTokens = false,
+             bool KeepSyntaxTree = false);
 
   void
   addImports(ArrayRef<std::pair<ModuleDecl::ImportedModule, ImportOptions>> IM);
@@ -1077,7 +1078,9 @@ public:
 
   ArrayRef<Token> getAllTokens() const;
 
-  bool shouldKeepSyntaxInfo() const;
+  bool shouldCollectToken() const;
+
+  bool shouldBuildSyntaxTree() const;
 
   syntax::SourceFileSyntax getSyntaxRoot() const;
   void setSyntaxRoot(syntax::SourceFileSyntax &&Root);

include/swift/Basic/LangOptions.h

@@ -248,9 +248,11 @@ namespace swift {
     /// This is used to guard preemptive testing for the fix-it.
     bool FixStringToSubstringConversions = false;
 
-    /// Whether to create and keep track of a libSyntax tree associated with
+    /// Whether collect tokens during parsing for syntax coloring.
-    /// this source file.
+    bool CollectParsedToken = false;
-    bool KeepSyntaxInfoInSourceFile = false;
+
+    /// Whether to parse syntax tree.
+    bool BuildSyntaxTree = false;
 
     /// Whether to verify the parsed syntax tree and emit related diagnostics.
     bool VerifySyntaxTree = false;

include/swift/Parse/Parser.h

@@ -201,11 +201,11 @@ public:
   Token Tok;
 
   /// \brief leading trivias for \c Tok.
-  /// Always empty if !SF.shouldKeepSyntaxInfo().
+  /// Always empty if !SF.shouldBuildSyntaxTree().
   syntax::Trivia LeadingTrivia;
 
   /// \brief trailing trivias for \c Tok.
-  /// Always empty if !SF.shouldKeepSyntaxInfo().
+  /// Always empty if !SF.shouldBuildSyntaxTree().
   syntax::Trivia TrailingTrivia;
 
   /// \brief The receiver to collect all consumed tokens.

lib/AST/Module.cpp

@@ -751,8 +751,10 @@ public:
 };
 
 struct SourceFile::SourceFileSyntaxInfo {
+  const bool Enable;
   /// The root of the syntax tree representing the source file.
   Optional<syntax::SourceFileSyntax> SyntaxRoot;
+  SourceFileSyntaxInfo(bool Enable): Enable(Enable) {}
 };
 
 bool SourceFile::hasSyntaxRoot() const {
@@ -1328,10 +1330,10 @@ static void performAutoImport(
 SourceFile::SourceFile(ModuleDecl &M, SourceFileKind K,
                        Optional<unsigned> bufferID,
                        ImplicitModuleImportKind ModImpKind,
-                       bool KeepSyntaxInfo)
+                       bool KeepParsedTokens, bool BuildSyntaxTree)
   : FileUnit(FileUnitKind::Source, M),
     BufferID(bufferID ? *bufferID : -1),
-    Kind(K), SyntaxInfo(*new SourceFileSyntaxInfo()) {
+    Kind(K), SyntaxInfo(*new SourceFileSyntaxInfo(BuildSyntaxTree)) {
   M.getASTContext().addDestructorCleanup(*this);
   performAutoImport(*this, ModImpKind);
 
@@ -1340,7 +1342,7 @@ SourceFile::SourceFile(ModuleDecl &M, SourceFileKind K,
     assert(!problem && "multiple main files?");
     (void)problem;
   }
-  if (KeepSyntaxInfo) {
+  if (KeepParsedTokens) {
     AllCorrectedTokens = std::vector<Token>();
   }
 }
@@ -1348,16 +1350,16 @@ SourceFile::SourceFile(ModuleDecl &M, SourceFileKind K,
 SourceFile::~SourceFile() { delete &SyntaxInfo; }
 
 std::vector<Token> &SourceFile::getTokenVector() {
-  assert(shouldKeepSyntaxInfo() && "Disabled");
+  assert(shouldCollectToken() && "Disabled");
   return *AllCorrectedTokens;
 }
 
 ArrayRef<Token> SourceFile::getAllTokens() const {
-  assert(shouldKeepSyntaxInfo() && "Disabled");
+  assert(shouldCollectToken() && "Disabled");
   return *AllCorrectedTokens;
 }
 
-bool SourceFile::shouldKeepSyntaxInfo() const {
+bool SourceFile::shouldCollectToken() const {
   switch (Kind) {
   case SourceFileKind::Library:
   case SourceFileKind::Main:
@@ -1368,6 +1370,17 @@ bool SourceFile::shouldKeepSyntaxInfo() const {
   }
 }
 
+bool SourceFile::shouldBuildSyntaxTree() const {
+  switch (Kind) {
+  case SourceFileKind::Library:
+  case SourceFileKind::Main:
+    return SyntaxInfo.Enable;
+  case SourceFileKind::REPL:
+  case SourceFileKind::SIL:
+    return false;
+  }
+}
+
 bool FileUnit::walk(ASTWalker &walker) {
   SmallVector<Decl *, 64> Decls;
   getTopLevelDecls(Decls);

lib/Frontend/Frontend.cpp

@@ -747,9 +747,10 @@ SourceFile *CompilerInstance::createSourceFileForMainModule(
     SourceFileKind fileKind, SourceFile::ImplicitModuleImportKind importKind,
     Optional<unsigned> bufferID) {
   ModuleDecl *mainModule = getMainModule();
-  bool keepSyntaxInfo = Invocation.getLangOptions().KeepSyntaxInfoInSourceFile;
   SourceFile *inputFile = new (*Context)
-      SourceFile(*mainModule, fileKind, bufferID, importKind, keepSyntaxInfo);
+      SourceFile(*mainModule, fileKind, bufferID, importKind,
+                 Invocation.getLangOptions().CollectParsedToken,
+                 Invocation.getLangOptions().BuildSyntaxTree);
   MainModule->addFile(*inputFile);
 
   if (bufferID && isPrimaryInput(*bufferID)) {

lib/FrontendTool/FrontendTool.cpp

@@ -838,7 +838,7 @@ static bool performCompile(CompilerInstance &Instance,
   FrontendOptions::ActionType Action = opts.RequestedAction;
 
   if (Action == FrontendOptions::ActionType::EmitSyntax) {
-    Instance.getASTContext().LangOpts.KeepSyntaxInfoInSourceFile = true;
+    Instance.getASTContext().LangOpts.BuildSyntaxTree = true;
     Instance.getASTContext().LangOpts.VerifySyntaxTree = true;
   }
 

lib/Parse/Parser.cpp

@@ -336,7 +336,7 @@ Parser::Parser(unsigned BufferID, SourceFile &SF, SILParserTUStateBase *SIL,
               SF.getASTContext().LangOpts.AttachCommentsToDecls
                   ? CommentRetentionMode::AttachToNextToken
                   : CommentRetentionMode::None,
-              SF.shouldKeepSyntaxInfo()
+              SF.shouldBuildSyntaxTree()
                   ? TriviaRetentionMode::WithTrivia
                   : TriviaRetentionMode::WithoutTrivia)),
           SF, SIL, PersistentState) {}
@@ -465,7 +465,7 @@ Parser::Parser(std::unique_ptr<Lexer> Lex, SourceFile &SF,
     SIL(SIL),
     CurDeclContext(&SF),
     Context(SF.getASTContext()),
-    TokReceiver(SF.shouldKeepSyntaxInfo() ?
+    TokReceiver(SF.shouldCollectToken() ?
                 new TokenRecorder(SF) :
                 new ConsumeTokenReceiver()),
     SyntaxContext(new SyntaxParsingContext(SyntaxContext, SF,
@@ -982,7 +982,8 @@ struct ParserUnit::Implementation {
             *ModuleDecl::create(Ctx.getIdentifier(ModuleName), Ctx),
             SourceFileKind::Main, BufferID,
             SourceFile::ImplicitModuleImportKind::None,
-            Opts.KeepSyntaxInfoInSourceFile)) {
+            Opts.CollectParsedToken,
+            Opts.BuildSyntaxTree)) {
   }
 };
 

lib/Parse/SyntaxParsingContext.cpp

@@ -39,7 +39,7 @@ SyntaxParsingContext::SyntaxParsingContext(SyntaxParsingContext *&CtxtHolder,
                                            BufferID)),
       CtxtHolder(CtxtHolder), Arena(SF.getASTContext().getSyntaxArena()),
       Storage(getRootData().Storage), Offset(0), Mode(AccumulationMode::Root),
-      Enabled(SF.shouldKeepSyntaxInfo()) {
+      Enabled(SF.shouldBuildSyntaxTree()) {
   CtxtHolder = this;
   Storage.reserve(128);
 }

lib/Sema/SourceLoader.cpp

@@ -132,7 +132,8 @@ ModuleDecl *SourceLoader::loadModule(SourceLoc importLoc,
 
   auto *importFile = new (Ctx) SourceFile(*importMod, SourceFileKind::Library,
                                           bufferID, implicitImportKind,
-                                          Ctx.LangOpts.KeepSyntaxInfoInSourceFile);
+                                          Ctx.LangOpts.CollectParsedToken,
+                                          Ctx.LangOpts.BuildSyntaxTree);
   importMod->addFile(*importFile);
 
   bool done;

tools/SourceKit/lib/SwiftLang/SwiftASTManager.cpp

@@ -442,7 +442,7 @@ bool SwiftASTManager::initCompilerInvocation(CompilerInvocation &Invocation,
   Invocation.setSerializedDiagnosticsPath(StringRef());
   Invocation.getLangOptions().AttachCommentsToDecls = true;
   Invocation.getLangOptions().DiagnosticsEditorMode = true;
-  Invocation.getLangOptions().KeepSyntaxInfoInSourceFile = true;
+  Invocation.getLangOptions().CollectParsedToken = true;
   auto &FrontendOpts = Invocation.getFrontendOptions();
   if (FrontendOpts.PlaygroundTransform) {
     // The playground instrumenter changes the AST in ways that disrupt the
@@ -817,7 +817,7 @@ ASTUnitRef ASTProducer::createASTUnit(SwiftASTManager::Implementation &MgrImpl,
   InvokRef->Impl.Opts.applyToSubstitutingInputs(
       Invocation, convertFileContentsToInputs(Contents));
 
-  Invocation.getLangOptions().KeepSyntaxInfoInSourceFile = true;
+  Invocation.getLangOptions().CollectParsedToken = true;
 
   if (CompIns.setup(Invocation)) {
     // FIXME: Report the diagnostic.

tools/SourceKit/lib/SwiftLang/SwiftEditor.cpp

@@ -1753,7 +1753,7 @@ void SwiftEditorDocument::updateSemaInfo() {
 }
 
 void SwiftEditorDocument::parse(ImmutableTextSnapshotRef Snapshot,
-                                SwiftLangSupport &Lang) {
+                                SwiftLangSupport &Lang, bool BuildSyntexTree) {
   llvm::sys::ScopedLock L(Impl.AccessMtx);
 
   assert(Impl.SemanticInfo && "Impl.SemanticInfo must be set");
@@ -1773,7 +1773,7 @@ void SwiftEditorDocument::parse(ImmutableTextSnapshotRef Snapshot,
     Lang.getASTManager().
       initCompilerInvocation(CompInv, Args, StringRef(), Error);
   }
-
+  CompInv.getLangOptions().BuildSyntaxTree = BuildSyntexTree;
   // Access to Impl.SyntaxInfo is guarded by Impl.AccessMtx
   Impl.SyntaxInfo.reset(
     new SwiftDocumentSyntaxInfo(CompInv, Snapshot, Args, Impl.FilePath));
@@ -2095,12 +2095,12 @@ void SwiftLangSupport::editorOpen(StringRef Name, llvm::MemoryBuffer *Buf,
                                   ArrayRef<const char *> Args) {
 
   ImmutableTextSnapshotRef Snapshot = nullptr;
-
+  const bool BuildSyntaxTree = Consumer.syntaxTreeEnabled();
   auto EditorDoc = EditorDocuments.getByUnresolvedName(Name);
   if (!EditorDoc) {
     EditorDoc = new SwiftEditorDocument(Name, *this);
     Snapshot = EditorDoc->initializeText(Buf, Args);
-    EditorDoc->parse(Snapshot, *this);
+    EditorDoc->parse(Snapshot, *this, BuildSyntaxTree);
     if (EditorDocuments.getOrUpdate(Name, *this, EditorDoc)) {
       // Document already exists, re-initialize it. This should only happen
       // if we get OPEN request while the previous document is not closed.
@@ -2113,7 +2113,7 @@ void SwiftLangSupport::editorOpen(StringRef Name, llvm::MemoryBuffer *Buf,
 
   if (!Snapshot) {
     Snapshot = EditorDoc->initializeText(Buf, Args);
-    EditorDoc->parse(Snapshot, *this);
+    EditorDoc->parse(Snapshot, *this, BuildSyntaxTree);
   }
 
   if (Consumer.needsSemanticInfo()) {
@@ -2161,7 +2161,7 @@ void SwiftLangSupport::editorReplaceText(StringRef Name, llvm::MemoryBuffer *Buf
     Snapshot = EditorDoc->replaceText(Offset, Length, Buf,
                                       Consumer.needsSemanticInfo());
     assert(Snapshot);
-    EditorDoc->parse(Snapshot, *this);
+    EditorDoc->parse(Snapshot, *this, Consumer.syntaxTreeEnabled());
     EditorDoc->readSyntaxInfo(Consumer);
   } else {
     Snapshot = EditorDoc->getLatestSnapshot();

tools/SourceKit/lib/SwiftLang/SwiftLangSupport.h

@@ -91,7 +91,8 @@ public:
 
   ImmutableTextSnapshotRef getLatestSnapshot() const;
 
-  void parse(ImmutableTextSnapshotRef Snapshot, SwiftLangSupport &Lang);
+  void parse(ImmutableTextSnapshotRef Snapshot, SwiftLangSupport &Lang,
+             bool BuildSyntaxTree);
   void readSyntaxInfo(EditorConsumer& consumer);
   void readSemanticInfo(ImmutableTextSnapshotRef Snapshot,
                         EditorConsumer& Consumer);

tools/driver/swift_format_main.cpp

@@ -55,7 +55,7 @@ private:
 public:
   FormatterDocument(std::unique_ptr<llvm::MemoryBuffer> Buffer) {
     // Formatting logic requires tokens on source file.
-    CompInv.getLangOptions().KeepSyntaxInfoInSourceFile = true;
+    CompInv.getLangOptions().CollectParsedToken = true;
     updateCode(std::move(Buffer));
   }
 

tools/swift-ide-test/swift-ide-test.cpp

@@ -965,7 +965,8 @@ static int doSyntaxColoring(const CompilerInvocation &InitInvok,
   PrintingDiagnosticConsumer PrintDiags;
   CI.addDiagnosticConsumer(&PrintDiags);
   Invocation.getLangOptions().Playground = Playground;
-  Invocation.getLangOptions().KeepSyntaxInfoInSourceFile = true;
+  Invocation.getLangOptions().CollectParsedToken = true;
+  Invocation.getLangOptions().BuildSyntaxTree = true;
   if (CI.setup(Invocation))
     return 1;
   if (!RunTypeChecker)
@@ -1178,7 +1179,8 @@ private:
 static int doStructureAnnotation(const CompilerInvocation &InitInvok,
                                  StringRef SourceFilename) {
   CompilerInvocation Invocation(InitInvok);
-  Invocation.getLangOptions().KeepSyntaxInfoInSourceFile = true;
+  Invocation.getLangOptions().BuildSyntaxTree = true;
+  Invocation.getLangOptions().CollectParsedToken = true;
   Invocation.getFrontendOptions().InputsAndOutputs.addInputFile(SourceFilename);
 
   CompilerInstance CI;
@@ -2695,7 +2697,8 @@ static int doPrintRangeInfo(const CompilerInvocation &InitInvok,
   CompilerInvocation Invocation(InitInvok);
   Invocation.getFrontendOptions().InputsAndOutputs.addInputFile(SourceFileName);
   Invocation.getLangOptions().DisableAvailabilityChecking = false;
-  Invocation.getLangOptions().KeepSyntaxInfoInSourceFile = true;
+  Invocation.getLangOptions().BuildSyntaxTree = true;
+  Invocation.getLangOptions().CollectParsedToken = true;
 
   CompilerInstance CI;
 
@@ -3017,7 +3020,8 @@ int main(int argc, char *argv[]) {
   InitInvok.setModuleName(options::ModuleName);
 
   InitInvok.setSDKPath(options::SDK);
-  InitInvok.getLangOptions().KeepSyntaxInfoInSourceFile = true;
+  InitInvok.getLangOptions().CollectParsedToken = true;
+  InitInvok.getLangOptions().BuildSyntaxTree = true;
   if (options::DisableObjCInterop) {
     InitInvok.getLangOptions().EnableObjCInterop = false;
   } else if (options::EnableObjCInterop) {

tools/swift-refactor/swift-refactor.cpp

@@ -236,7 +236,8 @@ int main(int argc, char *argv[]) {
   Invocation.getFrontendOptions().InputsAndOutputs.addInputFile(
       options::SourceFilename);
   Invocation.getLangOptions().AttachCommentsToDecls = true;
-  Invocation.getLangOptions().KeepSyntaxInfoInSourceFile = true;
+  Invocation.getLangOptions().CollectParsedToken = true;
+  Invocation.getLangOptions().BuildSyntaxTree = true;
 
   for (auto FileName : options::InputFilenames)
     Invocation.getFrontendOptions().InputsAndOutputs.addInputFile(FileName);

tools/swift-syntax-test/swift-syntax-test.cpp

@@ -144,7 +144,7 @@ SourceFile *getSourceFile(CompilerInstance &Instance,
                           StringRef InputFileName,
                           const char *MainExecutablePath) {
   CompilerInvocation Invocation;
-  Invocation.getLangOptions().KeepSyntaxInfoInSourceFile = true;
+  Invocation.getLangOptions().BuildSyntaxTree = true;
   Invocation.getLangOptions().VerifySyntaxTree = true;
   Invocation.getFrontendOptions().InputsAndOutputs.addInputFile(InputFileName);
   Invocation.setMainExecutablePath(