libSyntax: create a basic infrastructure for generating libSyntax entities by using Parser.

tools/swift-syntax-test/swift-syntax-test.cpp

@@ -42,6 +42,7 @@ enum class ActionType {
   FullLexRoundTrip,
   FullParseRoundTrip,
   SerializeRawTree,
+  ParserGen,
   None
 };
 
@@ -62,6 +63,10 @@ Action(llvm::cl::desc("Action (required):"),
                    "round-trip-parse",
                    "Parse the source file and print it back out for "
                    "comparing against the input"),
+        clEnumValN(ActionType::ParserGen,
+                   "parse-gen",
+                   "Parse the source file and print it back out for "
+                   "comparing against the input"),
         clEnumValN(ActionType::SerializeRawTree,
                    "serialize-raw-tree",
                    "Parse the source file and serialize the raw tree"
@@ -72,6 +77,7 @@ InputSourceFilename("input-source-filename",
                     llvm::cl::desc("Path to the input .swift file"));
 } // end namespace options
 
+namespace {
 int getTokensFromFile(unsigned BufferID,
                       LangOptions &LangOpts,
                       SourceManager &SourceMgr,
@@ -103,37 +109,27 @@ getTokensFromFile(const StringRef InputFilename,
 
 void anchorForGetMainExecutable() {}
 
-int getSyntaxTree(const char *MainExecutablePath,
-                  const StringRef InputFilename,
-                  CompilerInstance &Instance,
-                  llvm::SmallVectorImpl<syntax::Syntax> &TopLevelDecls,
-                  std::vector<std::pair<RC<syntax::RawTokenSyntax>,
-                              syntax::AbsolutePosition>> &Tokens) {
-  CompilerInvocation Invocation;
-  Invocation.addInputFilename(InputFilename);
 
+SourceFile *getSourceFile(CompilerInstance &Instance,
+                          StringRef InputFileName,
+                          const char *MainExecutablePath) {
+  CompilerInvocation Invocation;
+  Invocation.getLangOptions().KeepTokensInSourceFile = true;
+  Invocation.addInputFilename(InputFileName);
   Invocation.setMainExecutablePath(
     llvm::sys::fs::getMainExecutable(MainExecutablePath,
       reinterpret_cast<void *>(&anchorForGetMainExecutable)));
 
   Invocation.setModuleName("Test");
-
-  auto &SourceMgr = Instance.getSourceMgr();
-
   PrintingDiagnosticConsumer DiagPrinter;
   Instance.addDiagnosticConsumer(&DiagPrinter);
   if (Instance.setup(Invocation)) {
-    return EXIT_FAILURE;
+    return nullptr;
   }
 
   // First, parse the file normally and get the regular old AST.
   Instance.performParseOnly();
 
-  if (Instance.getDiags().hadAnyError()) {
-    return EXIT_FAILURE;
-  }
-
-  auto BufferID = Instance.getInputBufferIDs().back();
   SourceFile *SF = nullptr;
   for (auto Unit : Instance.getMainModule()->getFiles()) {
     SF = dyn_cast<SourceFile>(Unit);
@@ -142,9 +138,21 @@ int getSyntaxTree(const char *MainExecutablePath,
     }
   }
   assert(SF && "No source file");
+  return SF;
+}
+
+int getSyntaxTree(const char *MainExecutablePath,
+                  const StringRef InputFilename,
+                  CompilerInstance &Instance,
+                  llvm::SmallVectorImpl<syntax::Syntax> &TopLevelDecls,
+                  std::vector<std::pair<RC<syntax::RawTokenSyntax>,
+                              syntax::AbsolutePosition>> &Tokens) {
+  auto *SF = getSourceFile(Instance, InputFilename, MainExecutablePath);
+  auto &SourceMgr = Instance.getSourceMgr();
+  auto BufferID = Instance.getInputBufferIDs().back();
 
   // Retokenize the buffer with full fidelity
-  if (getTokensFromFile(BufferID, Invocation.getLangOptions(),
+  if (getTokensFromFile(BufferID, SF->getASTContext().LangOpts,
                         SourceMgr,
                         Instance.getDiags(), Tokens) == EXIT_FAILURE) {
     return EXIT_FAILURE;
@@ -260,6 +268,16 @@ int doSerializeRawTree(const char *MainExecutablePath,
   return EXIT_SUCCESS;
 }
 
+int dumpParserGen(const char *MainExecutablePath,
+                  const StringRef InputFileName) {
+  CompilerInstance Instance;
+  SourceFile *SF = getSourceFile(Instance, InputFileName, MainExecutablePath);
+  SF->getSyntaxRoot().print(llvm::outs());
+  return 0;
+}
+
+}// end of anonymous namespace
+
 int main(int argc, char *argv[]) {
   llvm::cl::ParseCommandLineOptions(argc, argv, "Swift Syntax Test\n");
 
@@ -288,6 +306,9 @@ int main(int argc, char *argv[]) {
   case ActionType::SerializeRawTree:
     ExitCode = doSerializeRawTree(argv[0], options::InputSourceFilename);
     break;
+  case ActionType::ParserGen:
+    ExitCode = dumpParserGen(argv[0], options::InputSourceFilename);
+    break;
   case ActionType::None:
     llvm::errs() << "an action is required\n";
     llvm::cl::PrintHelpMessage();