Add frontend flag to serialize Syntax tree (#11095)

* Add frontend flag to serialize Syntax tree

* Rename dump-serialized-syntax-tree to emit-syntax

include/swift/Frontend/FrontendOptions.h

@@ -161,6 +161,7 @@ public:
     Typecheck, ///< Parse and type-check only
     DumpParse, ///< Parse only and dump AST
     DumpInterfaceHash, ///< Parse and dump the interface token hash.
+    EmitSyntax, ///< Parse and dump Syntax tree as JSON
     DumpAST, ///< Parse, type-check, and dump AST
     PrintAST, ///< Parse, type-check, and pretty-print AST
 

include/swift/Option/FrontendOptions.td

@@ -400,6 +400,9 @@ def emit_sorted_sil : Flag<["-"], "emit-sorted-sil">,
   HelpText<"When printing SIL, print out all sil entities sorted by name to "
            "ease diffing">;
 
+def emit_syntax : Flag<["-"], "emit-syntax">,
+  HelpText<"Parse input file(s) and emit the Syntax tree(s) as JSON">, ModeOpt;
+
 def use_malloc : Flag<["-"], "use-malloc">,
   HelpText<"Allocate internal data structures using malloc "
            "(for memory debugging)">;

lib/Driver/Driver.cpp

@@ -1168,6 +1168,7 @@ void Driver::buildOutputInfo(const ToolChain &TC, const DerivedArgList &Args,
     case options::OPT_typecheck:
     case options::OPT_dump_parse:
     case options::OPT_dump_ast:
+    case options::OPT_emit_syntax:
     case options::OPT_print_ast:
     case options::OPT_dump_type_refinement_contexts:
     case options::OPT_dump_scope_maps:

lib/Frontend/CompilerInvocation.cpp

@@ -320,6 +320,8 @@ static bool ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args,
       Action = FrontendOptions::DumpParse;
     } else if (Opt.matches(OPT_dump_ast)) {
       Action = FrontendOptions::DumpAST;
+    } else if (Opt.matches(OPT_emit_syntax)) {
+      Action = FrontendOptions::EmitSyntax;
     } else if (Opt.matches(OPT_dump_scope_maps)) {
       Action = FrontendOptions::DumpScopeMaps;
 
@@ -531,6 +533,7 @@ static bool ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args,
     case FrontendOptions::DumpParse:
     case FrontendOptions::DumpInterfaceHash:
     case FrontendOptions::DumpAST:
+    case FrontendOptions::EmitSyntax:
     case FrontendOptions::PrintAST:
     case FrontendOptions::DumpScopeMaps:
     case FrontendOptions::DumpTypeRefinementContexts:
@@ -743,6 +746,7 @@ static bool ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args,
     case FrontendOptions::DumpParse:
     case FrontendOptions::DumpInterfaceHash:
     case FrontendOptions::DumpAST:
+    case FrontendOptions::EmitSyntax:
     case FrontendOptions::PrintAST:
     case FrontendOptions::DumpScopeMaps:
     case FrontendOptions::DumpTypeRefinementContexts:
@@ -774,6 +778,7 @@ static bool ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args,
     case FrontendOptions::DumpParse:
     case FrontendOptions::DumpInterfaceHash:
     case FrontendOptions::DumpAST:
+    case FrontendOptions::EmitSyntax:
     case FrontendOptions::PrintAST:
     case FrontendOptions::EmitPCH:
     case FrontendOptions::DumpScopeMaps:
@@ -806,6 +811,7 @@ static bool ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args,
     case FrontendOptions::DumpParse:
     case FrontendOptions::DumpInterfaceHash:
     case FrontendOptions::DumpAST:
+    case FrontendOptions::EmitSyntax:
     case FrontendOptions::PrintAST:
     case FrontendOptions::DumpScopeMaps:
     case FrontendOptions::DumpTypeRefinementContexts:
@@ -840,6 +846,7 @@ static bool ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args,
     case FrontendOptions::DumpParse:
     case FrontendOptions::DumpInterfaceHash:
     case FrontendOptions::DumpAST:
+    case FrontendOptions::EmitSyntax:
     case FrontendOptions::PrintAST:
     case FrontendOptions::EmitPCH:
     case FrontendOptions::DumpScopeMaps:

lib/Frontend/FrontendOptions.cpp

@@ -23,6 +23,7 @@ bool FrontendOptions::actionHasOutput() const {
   case Typecheck:
   case DumpParse:
   case DumpAST:
+  case EmitSyntax:
   case DumpInterfaceHash:
   case PrintAST:
   case DumpScopeMaps:
@@ -56,6 +57,7 @@ bool FrontendOptions::actionIsImmediate() const {
   case Typecheck:
   case DumpParse:
   case DumpAST:
+  case EmitSyntax:
   case DumpInterfaceHash:
   case PrintAST:
   case DumpScopeMaps:

lib/FrontendTool/FrontendTool.cpp

@@ -32,6 +32,7 @@
 #include "swift/AST/DiagnosticsSema.h"
 #include "swift/AST/IRGenOptions.h"
 #include "swift/AST/ASTMangler.h"
+#include "swift/AST/LegacyASTTransformer.h"
 #include "swift/AST/ReferencedNameTracker.h"
 #include "swift/AST/TypeRefinementContext.h"
 #include "swift/Basic/Dwarf.h"
@@ -56,6 +57,7 @@
 #include "swift/Serialization/SerializationOptions.h"
 #include "swift/Serialization/SerializedModuleLoader.h"
 #include "swift/SILOptimizer/PassManager/Passes.h"
+#include "swift/Syntax/Serialization/SyntaxSerialization.h"
 
 // FIXME: We're just using CompilerInstance::createOutputFile.
 // This API should be sunk down to LLVM.
@@ -228,17 +230,72 @@ static bool emitLoadedModuleTrace(ASTContext &ctxt,
   return true;
 }
 
+
+/// Gets an output stream for the provided output filename, or diagnoses to the
+/// provided AST Context and returns null if there was an error getting the
+/// stream.
+static std::unique_ptr<llvm::raw_fd_ostream>
+getFileOutputStream(StringRef OutputFilename, ASTContext &Ctx) {
+  std::error_code errorCode;
+  auto os = llvm::make_unique<llvm::raw_fd_ostream>(
+              OutputFilename, errorCode, llvm::sys::fs::F_None);
+  if (errorCode) {
+    Ctx.Diags.diagnose(SourceLoc(), diag::error_opening_output,
+                       OutputFilename, errorCode.message());
+    return nullptr;
+  }
+  return os;
+}
+
+/// Writes the Syntax tree to the given file
+static bool emitSyntax(SourceFile *SF, LangOptions &LangOpts,
+                       SourceManager &SM, StringRef OutputFilename) {
+  auto bufferID = SF->getBufferID();
+  assert(bufferID && "frontend should have a buffer ID "
+         "for the main source file");
+
+  // Get a full token stream with associated Trivia.
+  syntax::TokenPositionList tokens =
+    tokenizeWithTrivia(LangOpts, SM, *bufferID);
+
+  llvm::SmallVector<Decl *, 16> topLevelDecls;
+  SF->getTopLevelDecls(topLevelDecls);
+
+  // Convert the old ASTs to the Syntax tree and print
+  // them out.
+  SyntaxASTMap ASTMap;
+  std::vector<RC<syntax::RawSyntax>> topLevelRaw;
+  for (auto *decl : topLevelDecls) {
+    if (decl->escapedFromIfConfig()) {
+      continue;
+    }
+    auto newNode = transformAST(ASTNode(decl), ASTMap, SM, *bufferID, tokens);
+    if (newNode.hasValue()) {
+      topLevelRaw.push_back(newNode->getRaw());
+    }
+  }
+
+  // Push the EOF token -- this ensures that any remaining trivia in the
+  // file is serialized as the EOF's leading trivia.
+  if (!tokens.empty() && tokens.back().first->getTokenKind() == tok::eof) {
+    topLevelRaw.push_back(tokens.back().first);
+  }
+
+  auto os = getFileOutputStream(OutputFilename, SF->getASTContext());
+  if (!os) return true;
+
+  json::Output jsonOut(*os);
+  jsonOut << topLevelRaw;
+  *os << "\n";
+  return false;
+}
+
 /// Writes SIL out to the given file.
 static bool writeSIL(SILModule &SM, ModuleDecl *M, bool EmitVerboseSIL,
                      StringRef OutputFilename, bool SortSIL) {
-  std::error_code EC;
-  llvm::raw_fd_ostream OS(OutputFilename, EC, llvm::sys::fs::F_None);
-  if (EC) {
-    M->getASTContext().Diags.diagnose(SourceLoc(), diag::error_opening_output,
-                                      OutputFilename, EC.message());
-    return true;
-  }
-  SM.print(OS, EmitVerboseSIL, M, SortSIL);
+  auto OS = getFileOutputStream(OutputFilename, M->getASTContext());
+  if (!OS) return true;
+  SM.print(*OS, EmitVerboseSIL, M, SortSIL);
   return false;
 }
 
@@ -568,6 +625,7 @@ static bool performCompile(CompilerInstance &Instance,
   // so dump or print the main source file and return.
   if (Action == FrontendOptions::DumpParse ||
       Action == FrontendOptions::DumpAST ||
+      Action == FrontendOptions::EmitSyntax ||
       Action == FrontendOptions::PrintAST ||
       Action == FrontendOptions::DumpScopeMaps ||
       Action == FrontendOptions::DumpTypeRefinementContexts ||
@@ -627,7 +685,10 @@ static bool performCompile(CompilerInstance &Instance,
       SF->getTypeRefinementContext()->dump(llvm::errs(), Context.SourceMgr);
     else if (Action == FrontendOptions::DumpInterfaceHash)
       SF->dumpInterfaceHash(llvm::errs());
-    else
+    else if (Action == FrontendOptions::EmitSyntax) {
+      emitSyntax(SF, Invocation.getLangOptions(), Instance.getSourceMgr(),
+                 opts.getSingleOutputFilename());
+    } else
       SF->dump();
     return Context.hadError();
   } else if (Action == FrontendOptions::EmitImportedModules) {

test/Frontend/emit-syntax.swift

@@ -0,0 +1,20 @@
+// RUN: %target-swift-frontend -emit-syntax %s | %FileCheck %s
+
+// CHECK: "kind": "kw_struct"
+// CHECK: "kind": "identifier",
+// CHECK: "text": "Foo"
+// CHECK: "kind": "l_brace"
+struct Foo {
+  // CHECK: "kind": "kw_let"
+  // CHECK: "kind": "colon"
+  // CHECK: "kind": "identifier"
+  // CHECK: "text": "Int"
+  let x: Int
+// CHECK: "kind": "r_brace"
+}
+
+// CHECK: "leadingTrivia": [
+// CHECK: "kind": "LineComment",
+// CHECK: "value": "\/\/ Comment at the end of the file"
+
+// Comment at the end of the file