diff --git a/include/swift/AST/Module.h b/include/swift/AST/Module.h index 839a3d30a77..07ac6633d8e 100644 --- a/include/swift/AST/Module.h +++ b/include/swift/AST/Module.h @@ -888,7 +888,8 @@ public: ASTStage_t ASTStage = Parsing; SourceFile(ModuleDecl &M, SourceFileKind K, Optional bufferID, - ImplicitModuleImportKind ModImpKind, bool KeepSyntaxInfo); + ImplicitModuleImportKind ModImpKind, bool KeepParsedTokens = false, + bool KeepSyntaxTree = false); void addImports(ArrayRef> IM); @@ -1077,7 +1078,9 @@ public: ArrayRef getAllTokens() const; - bool shouldKeepSyntaxInfo() const; + bool shouldCollectToken() const; + + bool shouldBuildSyntaxTree() const; syntax::SourceFileSyntax getSyntaxRoot() const; void setSyntaxRoot(syntax::SourceFileSyntax &&Root); diff --git a/include/swift/Basic/LangOptions.h b/include/swift/Basic/LangOptions.h index 2bcf7c55286..81fbc0ccfa7 100644 --- a/include/swift/Basic/LangOptions.h +++ b/include/swift/Basic/LangOptions.h @@ -248,9 +248,11 @@ namespace swift { /// This is used to guard preemptive testing for the fix-it. bool FixStringToSubstringConversions = false; - /// Whether to create and keep track of a libSyntax tree associated with - /// this source file. - bool KeepSyntaxInfoInSourceFile = false; + /// Whether collect tokens during parsing for syntax coloring. + bool CollectParsedToken = false; + + /// Whether to parse syntax tree. + bool BuildSyntaxTree = false; /// Whether to verify the parsed syntax tree and emit related diagnostics. bool VerifySyntaxTree = false; diff --git a/include/swift/Parse/Parser.h b/include/swift/Parse/Parser.h index 09786f530ea..6e67e55a2d9 100644 --- a/include/swift/Parse/Parser.h +++ b/include/swift/Parse/Parser.h @@ -201,11 +201,11 @@ public: Token Tok; /// \brief leading trivias for \c Tok. - /// Always empty if !SF.shouldKeepSyntaxInfo(). + /// Always empty if !SF.shouldBuildSyntaxTree(). syntax::Trivia LeadingTrivia; /// \brief trailing trivias for \c Tok. - /// Always empty if !SF.shouldKeepSyntaxInfo(). + /// Always empty if !SF.shouldBuildSyntaxTree(). syntax::Trivia TrailingTrivia; /// \brief The receiver to collect all consumed tokens. diff --git a/lib/AST/Module.cpp b/lib/AST/Module.cpp index 76808e173cc..dccc06dd06c 100644 --- a/lib/AST/Module.cpp +++ b/lib/AST/Module.cpp @@ -751,8 +751,10 @@ public: }; struct SourceFile::SourceFileSyntaxInfo { + const bool Enable; /// The root of the syntax tree representing the source file. Optional SyntaxRoot; + SourceFileSyntaxInfo(bool Enable): Enable(Enable) {} }; bool SourceFile::hasSyntaxRoot() const { @@ -1328,10 +1330,10 @@ static void performAutoImport( SourceFile::SourceFile(ModuleDecl &M, SourceFileKind K, Optional bufferID, ImplicitModuleImportKind ModImpKind, - bool KeepSyntaxInfo) + bool KeepParsedTokens, bool BuildSyntaxTree) : FileUnit(FileUnitKind::Source, M), BufferID(bufferID ? *bufferID : -1), - Kind(K), SyntaxInfo(*new SourceFileSyntaxInfo()) { + Kind(K), SyntaxInfo(*new SourceFileSyntaxInfo(BuildSyntaxTree)) { M.getASTContext().addDestructorCleanup(*this); performAutoImport(*this, ModImpKind); @@ -1340,7 +1342,7 @@ SourceFile::SourceFile(ModuleDecl &M, SourceFileKind K, assert(!problem && "multiple main files?"); (void)problem; } - if (KeepSyntaxInfo) { + if (KeepParsedTokens) { AllCorrectedTokens = std::vector(); } } @@ -1348,16 +1350,16 @@ SourceFile::SourceFile(ModuleDecl &M, SourceFileKind K, SourceFile::~SourceFile() { delete &SyntaxInfo; } std::vector &SourceFile::getTokenVector() { - assert(shouldKeepSyntaxInfo() && "Disabled"); + assert(shouldCollectToken() && "Disabled"); return *AllCorrectedTokens; } ArrayRef SourceFile::getAllTokens() const { - assert(shouldKeepSyntaxInfo() && "Disabled"); + assert(shouldCollectToken() && "Disabled"); return *AllCorrectedTokens; } -bool SourceFile::shouldKeepSyntaxInfo() const { +bool SourceFile::shouldCollectToken() const { switch (Kind) { case SourceFileKind::Library: case SourceFileKind::Main: @@ -1368,6 +1370,17 @@ bool SourceFile::shouldKeepSyntaxInfo() const { } } +bool SourceFile::shouldBuildSyntaxTree() const { + switch (Kind) { + case SourceFileKind::Library: + case SourceFileKind::Main: + return SyntaxInfo.Enable; + case SourceFileKind::REPL: + case SourceFileKind::SIL: + return false; + } +} + bool FileUnit::walk(ASTWalker &walker) { SmallVector Decls; getTopLevelDecls(Decls); diff --git a/lib/Frontend/Frontend.cpp b/lib/Frontend/Frontend.cpp index 1ebb703a5df..ae1ad5654b2 100644 --- a/lib/Frontend/Frontend.cpp +++ b/lib/Frontend/Frontend.cpp @@ -747,9 +747,10 @@ SourceFile *CompilerInstance::createSourceFileForMainModule( SourceFileKind fileKind, SourceFile::ImplicitModuleImportKind importKind, Optional bufferID) { ModuleDecl *mainModule = getMainModule(); - bool keepSyntaxInfo = Invocation.getLangOptions().KeepSyntaxInfoInSourceFile; SourceFile *inputFile = new (*Context) - SourceFile(*mainModule, fileKind, bufferID, importKind, keepSyntaxInfo); + SourceFile(*mainModule, fileKind, bufferID, importKind, + Invocation.getLangOptions().CollectParsedToken, + Invocation.getLangOptions().BuildSyntaxTree); MainModule->addFile(*inputFile); if (bufferID && isPrimaryInput(*bufferID)) { diff --git a/lib/FrontendTool/FrontendTool.cpp b/lib/FrontendTool/FrontendTool.cpp index 7670bddcb5a..2aa2f66870c 100644 --- a/lib/FrontendTool/FrontendTool.cpp +++ b/lib/FrontendTool/FrontendTool.cpp @@ -838,7 +838,7 @@ static bool performCompile(CompilerInstance &Instance, FrontendOptions::ActionType Action = opts.RequestedAction; if (Action == FrontendOptions::ActionType::EmitSyntax) { - Instance.getASTContext().LangOpts.KeepSyntaxInfoInSourceFile = true; + Instance.getASTContext().LangOpts.BuildSyntaxTree = true; Instance.getASTContext().LangOpts.VerifySyntaxTree = true; } diff --git a/lib/Parse/Parser.cpp b/lib/Parse/Parser.cpp index 779b0b1234e..5044e91f0c2 100644 --- a/lib/Parse/Parser.cpp +++ b/lib/Parse/Parser.cpp @@ -336,7 +336,7 @@ Parser::Parser(unsigned BufferID, SourceFile &SF, SILParserTUStateBase *SIL, SF.getASTContext().LangOpts.AttachCommentsToDecls ? CommentRetentionMode::AttachToNextToken : CommentRetentionMode::None, - SF.shouldKeepSyntaxInfo() + SF.shouldBuildSyntaxTree() ? TriviaRetentionMode::WithTrivia : TriviaRetentionMode::WithoutTrivia)), SF, SIL, PersistentState) {} @@ -465,7 +465,7 @@ Parser::Parser(std::unique_ptr Lex, SourceFile &SF, SIL(SIL), CurDeclContext(&SF), Context(SF.getASTContext()), - TokReceiver(SF.shouldKeepSyntaxInfo() ? + TokReceiver(SF.shouldCollectToken() ? new TokenRecorder(SF) : new ConsumeTokenReceiver()), SyntaxContext(new SyntaxParsingContext(SyntaxContext, SF, @@ -982,7 +982,8 @@ struct ParserUnit::Implementation { *ModuleDecl::create(Ctx.getIdentifier(ModuleName), Ctx), SourceFileKind::Main, BufferID, SourceFile::ImplicitModuleImportKind::None, - Opts.KeepSyntaxInfoInSourceFile)) { + Opts.CollectParsedToken, + Opts.BuildSyntaxTree)) { } }; diff --git a/lib/Parse/SyntaxParsingContext.cpp b/lib/Parse/SyntaxParsingContext.cpp index c17400e695c..b0720693e49 100644 --- a/lib/Parse/SyntaxParsingContext.cpp +++ b/lib/Parse/SyntaxParsingContext.cpp @@ -39,7 +39,7 @@ SyntaxParsingContext::SyntaxParsingContext(SyntaxParsingContext *&CtxtHolder, BufferID)), CtxtHolder(CtxtHolder), Arena(SF.getASTContext().getSyntaxArena()), Storage(getRootData().Storage), Offset(0), Mode(AccumulationMode::Root), - Enabled(SF.shouldKeepSyntaxInfo()) { + Enabled(SF.shouldBuildSyntaxTree()) { CtxtHolder = this; Storage.reserve(128); } diff --git a/lib/Sema/SourceLoader.cpp b/lib/Sema/SourceLoader.cpp index 9d1cfa4ab5e..e349a1be298 100644 --- a/lib/Sema/SourceLoader.cpp +++ b/lib/Sema/SourceLoader.cpp @@ -132,7 +132,8 @@ ModuleDecl *SourceLoader::loadModule(SourceLoc importLoc, auto *importFile = new (Ctx) SourceFile(*importMod, SourceFileKind::Library, bufferID, implicitImportKind, - Ctx.LangOpts.KeepSyntaxInfoInSourceFile); + Ctx.LangOpts.CollectParsedToken, + Ctx.LangOpts.BuildSyntaxTree); importMod->addFile(*importFile); bool done; diff --git a/tools/SourceKit/lib/SwiftLang/SwiftASTManager.cpp b/tools/SourceKit/lib/SwiftLang/SwiftASTManager.cpp index 3cd14385b49..5b2cd8ad1bb 100644 --- a/tools/SourceKit/lib/SwiftLang/SwiftASTManager.cpp +++ b/tools/SourceKit/lib/SwiftLang/SwiftASTManager.cpp @@ -442,7 +442,7 @@ bool SwiftASTManager::initCompilerInvocation(CompilerInvocation &Invocation, Invocation.setSerializedDiagnosticsPath(StringRef()); Invocation.getLangOptions().AttachCommentsToDecls = true; Invocation.getLangOptions().DiagnosticsEditorMode = true; - Invocation.getLangOptions().KeepSyntaxInfoInSourceFile = true; + Invocation.getLangOptions().CollectParsedToken = true; auto &FrontendOpts = Invocation.getFrontendOptions(); if (FrontendOpts.PlaygroundTransform) { // The playground instrumenter changes the AST in ways that disrupt the @@ -817,7 +817,7 @@ ASTUnitRef ASTProducer::createASTUnit(SwiftASTManager::Implementation &MgrImpl, InvokRef->Impl.Opts.applyToSubstitutingInputs( Invocation, convertFileContentsToInputs(Contents)); - Invocation.getLangOptions().KeepSyntaxInfoInSourceFile = true; + Invocation.getLangOptions().CollectParsedToken = true; if (CompIns.setup(Invocation)) { // FIXME: Report the diagnostic. diff --git a/tools/SourceKit/lib/SwiftLang/SwiftEditor.cpp b/tools/SourceKit/lib/SwiftLang/SwiftEditor.cpp index ea4270b883a..28dbc045487 100644 --- a/tools/SourceKit/lib/SwiftLang/SwiftEditor.cpp +++ b/tools/SourceKit/lib/SwiftLang/SwiftEditor.cpp @@ -1753,7 +1753,7 @@ void SwiftEditorDocument::updateSemaInfo() { } void SwiftEditorDocument::parse(ImmutableTextSnapshotRef Snapshot, - SwiftLangSupport &Lang) { + SwiftLangSupport &Lang, bool BuildSyntexTree) { llvm::sys::ScopedLock L(Impl.AccessMtx); assert(Impl.SemanticInfo && "Impl.SemanticInfo must be set"); @@ -1773,7 +1773,7 @@ void SwiftEditorDocument::parse(ImmutableTextSnapshotRef Snapshot, Lang.getASTManager(). initCompilerInvocation(CompInv, Args, StringRef(), Error); } - + CompInv.getLangOptions().BuildSyntaxTree = BuildSyntexTree; // Access to Impl.SyntaxInfo is guarded by Impl.AccessMtx Impl.SyntaxInfo.reset( new SwiftDocumentSyntaxInfo(CompInv, Snapshot, Args, Impl.FilePath)); @@ -2095,12 +2095,12 @@ void SwiftLangSupport::editorOpen(StringRef Name, llvm::MemoryBuffer *Buf, ArrayRef Args) { ImmutableTextSnapshotRef Snapshot = nullptr; - + const bool BuildSyntaxTree = Consumer.syntaxTreeEnabled(); auto EditorDoc = EditorDocuments.getByUnresolvedName(Name); if (!EditorDoc) { EditorDoc = new SwiftEditorDocument(Name, *this); Snapshot = EditorDoc->initializeText(Buf, Args); - EditorDoc->parse(Snapshot, *this); + EditorDoc->parse(Snapshot, *this, BuildSyntaxTree); if (EditorDocuments.getOrUpdate(Name, *this, EditorDoc)) { // Document already exists, re-initialize it. This should only happen // if we get OPEN request while the previous document is not closed. @@ -2113,7 +2113,7 @@ void SwiftLangSupport::editorOpen(StringRef Name, llvm::MemoryBuffer *Buf, if (!Snapshot) { Snapshot = EditorDoc->initializeText(Buf, Args); - EditorDoc->parse(Snapshot, *this); + EditorDoc->parse(Snapshot, *this, BuildSyntaxTree); } if (Consumer.needsSemanticInfo()) { @@ -2161,7 +2161,7 @@ void SwiftLangSupport::editorReplaceText(StringRef Name, llvm::MemoryBuffer *Buf Snapshot = EditorDoc->replaceText(Offset, Length, Buf, Consumer.needsSemanticInfo()); assert(Snapshot); - EditorDoc->parse(Snapshot, *this); + EditorDoc->parse(Snapshot, *this, Consumer.syntaxTreeEnabled()); EditorDoc->readSyntaxInfo(Consumer); } else { Snapshot = EditorDoc->getLatestSnapshot(); diff --git a/tools/SourceKit/lib/SwiftLang/SwiftLangSupport.h b/tools/SourceKit/lib/SwiftLang/SwiftLangSupport.h index ba60ae5dbab..2b044dedafa 100644 --- a/tools/SourceKit/lib/SwiftLang/SwiftLangSupport.h +++ b/tools/SourceKit/lib/SwiftLang/SwiftLangSupport.h @@ -91,7 +91,8 @@ public: ImmutableTextSnapshotRef getLatestSnapshot() const; - void parse(ImmutableTextSnapshotRef Snapshot, SwiftLangSupport &Lang); + void parse(ImmutableTextSnapshotRef Snapshot, SwiftLangSupport &Lang, + bool BuildSyntaxTree); void readSyntaxInfo(EditorConsumer& consumer); void readSemanticInfo(ImmutableTextSnapshotRef Snapshot, EditorConsumer& Consumer); diff --git a/tools/driver/swift_format_main.cpp b/tools/driver/swift_format_main.cpp index c9a74e6161e..a641fe4e0f2 100644 --- a/tools/driver/swift_format_main.cpp +++ b/tools/driver/swift_format_main.cpp @@ -55,7 +55,7 @@ private: public: FormatterDocument(std::unique_ptr Buffer) { // Formatting logic requires tokens on source file. - CompInv.getLangOptions().KeepSyntaxInfoInSourceFile = true; + CompInv.getLangOptions().CollectParsedToken = true; updateCode(std::move(Buffer)); } diff --git a/tools/swift-ide-test/swift-ide-test.cpp b/tools/swift-ide-test/swift-ide-test.cpp index e740da17525..3057489bc61 100644 --- a/tools/swift-ide-test/swift-ide-test.cpp +++ b/tools/swift-ide-test/swift-ide-test.cpp @@ -965,7 +965,8 @@ static int doSyntaxColoring(const CompilerInvocation &InitInvok, PrintingDiagnosticConsumer PrintDiags; CI.addDiagnosticConsumer(&PrintDiags); Invocation.getLangOptions().Playground = Playground; - Invocation.getLangOptions().KeepSyntaxInfoInSourceFile = true; + Invocation.getLangOptions().CollectParsedToken = true; + Invocation.getLangOptions().BuildSyntaxTree = true; if (CI.setup(Invocation)) return 1; if (!RunTypeChecker) @@ -1178,7 +1179,8 @@ private: static int doStructureAnnotation(const CompilerInvocation &InitInvok, StringRef SourceFilename) { CompilerInvocation Invocation(InitInvok); - Invocation.getLangOptions().KeepSyntaxInfoInSourceFile = true; + Invocation.getLangOptions().BuildSyntaxTree = true; + Invocation.getLangOptions().CollectParsedToken = true; Invocation.getFrontendOptions().InputsAndOutputs.addInputFile(SourceFilename); CompilerInstance CI; @@ -2695,7 +2697,8 @@ static int doPrintRangeInfo(const CompilerInvocation &InitInvok, CompilerInvocation Invocation(InitInvok); Invocation.getFrontendOptions().InputsAndOutputs.addInputFile(SourceFileName); Invocation.getLangOptions().DisableAvailabilityChecking = false; - Invocation.getLangOptions().KeepSyntaxInfoInSourceFile = true; + Invocation.getLangOptions().BuildSyntaxTree = true; + Invocation.getLangOptions().CollectParsedToken = true; CompilerInstance CI; @@ -3017,7 +3020,8 @@ int main(int argc, char *argv[]) { InitInvok.setModuleName(options::ModuleName); InitInvok.setSDKPath(options::SDK); - InitInvok.getLangOptions().KeepSyntaxInfoInSourceFile = true; + InitInvok.getLangOptions().CollectParsedToken = true; + InitInvok.getLangOptions().BuildSyntaxTree = true; if (options::DisableObjCInterop) { InitInvok.getLangOptions().EnableObjCInterop = false; } else if (options::EnableObjCInterop) { diff --git a/tools/swift-refactor/swift-refactor.cpp b/tools/swift-refactor/swift-refactor.cpp index 8cc3d9687f8..71ebf2f483e 100644 --- a/tools/swift-refactor/swift-refactor.cpp +++ b/tools/swift-refactor/swift-refactor.cpp @@ -236,7 +236,8 @@ int main(int argc, char *argv[]) { Invocation.getFrontendOptions().InputsAndOutputs.addInputFile( options::SourceFilename); Invocation.getLangOptions().AttachCommentsToDecls = true; - Invocation.getLangOptions().KeepSyntaxInfoInSourceFile = true; + Invocation.getLangOptions().CollectParsedToken = true; + Invocation.getLangOptions().BuildSyntaxTree = true; for (auto FileName : options::InputFilenames) Invocation.getFrontendOptions().InputsAndOutputs.addInputFile(FileName); diff --git a/tools/swift-syntax-test/swift-syntax-test.cpp b/tools/swift-syntax-test/swift-syntax-test.cpp index ec6627358b7..b075dc6ad9d 100644 --- a/tools/swift-syntax-test/swift-syntax-test.cpp +++ b/tools/swift-syntax-test/swift-syntax-test.cpp @@ -144,7 +144,7 @@ SourceFile *getSourceFile(CompilerInstance &Instance, StringRef InputFileName, const char *MainExecutablePath) { CompilerInvocation Invocation; - Invocation.getLangOptions().KeepSyntaxInfoInSourceFile = true; + Invocation.getLangOptions().BuildSyntaxTree = true; Invocation.getLangOptions().VerifySyntaxTree = true; Invocation.getFrontendOptions().InputsAndOutputs.addInputFile(InputFileName); Invocation.setMainExecutablePath(