Add serialization code for RawSyntax

This patch will allow for serialization of RawSyntax trees to JSON,
which allows external tools to get access to a RawSyntax tree.

This also adds a hook into swift-syntax-test to generate JSON for a
given Swift source file, which will be used in tests in subsequent
commits.

include/swift/Syntax/TokenKinds.def

@@ -24,6 +24,8 @@
 ///   POUND_OLD_OBJECT_LITERAL(kw, new_kw, old_arg, new_arg)
 ///   POUND_CONFIG(kw)
 /// PUNCTUATOR(name, str)
+/// LITERAL(name)
+/// MISC(name)
 ///
 //===----------------------------------------------------------------------===//
 
@@ -108,6 +110,18 @@
 #define PUNCTUATOR(name, str)
 #endif
 
+/// LITERAL(name)
+///   Tokens representing literal values, e.g. 'integer_literal'.
+#ifndef LITERAL
+#define LITERAL(name)
+#endif
+
+/// MISC(name)
+///   Miscellaneous tokens, e.g. 'eof' and 'unknown'.
+#ifndef MISC
+#define MISC(name)
+#endif
+
 // Keywords that start decls.
 DECL_KEYWORD(associatedtype)
 DECL_KEYWORD(class)
@@ -253,6 +267,23 @@ POUND_KEYWORD(column)
 POUND_KEYWORD(function)
 POUND_KEYWORD(dsohandle)
 
+// Single-token literals
+LITERAL(integer_literal)
+LITERAL(floating_literal)
+LITERAL(string_literal)
+
+// Miscellaneous tokens.
+MISC(unknown)
+MISC(eof)
+MISC(code_complete)
+MISC(identifier)
+MISC(oper_binary_unspaced)   // "x+y"
+MISC(oper_binary_spaced)     // "x + y"
+MISC(oper_postfix)
+MISC(oper_prefix)
+MISC(dollarident)
+MISC(sil_local_name)       // %42 in SIL mode.
+MISC(comment)
 
 #undef KEYWORD
 #undef SWIFT_KEYWORD
@@ -266,3 +297,5 @@ POUND_KEYWORD(dsohandle)
 #undef POUND_OLD_OBJECT_LITERAL
 #undef POUND_CONFIG
 #undef PUNCTUATOR
+#undef LITERAL
+#undef MISC

include/swift/Syntax/TokenKinds.h

@@ -19,21 +19,8 @@
 
 namespace swift {
 enum class tok {
-  unknown = 0,
-  eof,
-  code_complete,
-  identifier,
-  oper_binary_unspaced,   // "x+y"
-  oper_binary_spaced,     // "x + y"
-  oper_postfix,
-  oper_prefix,
-  dollarident,
-  integer_literal,
-  floating_literal,
-  string_literal,
-  sil_local_name,      // %42 in SIL mode.
-  comment,
-
+#define LITERAL(X) X,
+#define MISC(X) X,
 #define KEYWORD(X) kw_ ## X,
 #define PUNCTUATOR(X, Y) X,
 #define POUND_KEYWORD(X) pound_ ## X,

lib/Syntax/LegacyASTTransformer.cpp

@@ -161,7 +161,7 @@ RC<SyntaxData> LegacyASTTransformer::getUnknownDecl(Decl *D) {
   std::copy(ComprisingTokens.begin(),
             ComprisingTokens.end(),
             std::back_inserter(Layout));
-  auto Raw = RawSyntax::make(SyntaxKind::UnknownExpr,
+  auto Raw = RawSyntax::make(SyntaxKind::UnknownDecl,
                              Layout,
                              SourcePresence::Present);
   return UnknownDeclSyntaxData::make(Raw);
@@ -175,7 +175,7 @@ RC<SyntaxData> LegacyASTTransformer::getUnknownStmt(Stmt *S) {
   std::copy(ComprisingTokens.begin(),
             ComprisingTokens.end(),
             std::back_inserter(Layout));
-  auto Raw = RawSyntax::make(SyntaxKind::UnknownExpr,
+  auto Raw = RawSyntax::make(SyntaxKind::UnknownStmt,
                              Layout,
                              SourcePresence::Present);
   return UnknownStmtSyntaxData::make(Raw);

tools/swift-syntax-test/swift-syntax-test.cpp

@@ -27,6 +27,7 @@
 #include "swift/Parse/Lexer.h"
 #include "swift/Subsystems.h"
 #include "swift/Syntax/LegacyASTTransformer.h"
+#include "swift/Syntax/Serialization/SyntaxSerialization.h"
 #include "swift/Syntax/SyntaxData.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/FileSystem.h"
@@ -39,6 +40,7 @@ enum class ActionType {
   DumpTokenSyntax,
   FullLexRoundTrip,
   FullParseRoundTrip,
+  SerializeRawTree,
   None
 };
 
@@ -58,7 +60,11 @@ Action(llvm::cl::desc("Action (required):"),
         clEnumValN(ActionType::FullParseRoundTrip,
                    "round-trip-parse",
                    "Parse the source file and print it back out for "
-                   "comparing against the input")));
+                   "comparing against the input"),
+        clEnumValN(ActionType::SerializeRawTree,
+                   "serialize-raw-tree",
+                   "Parse the source file and serialize the raw tree"
+                   "to JSON")));
 
 static llvm::cl::opt<std::string>
 InputSourceFilename("input-source-filename",
@@ -94,53 +100,10 @@ getTokensFromFile(const StringRef InputFilename,
   return getTokensFromFile(BufferID, LangOpts, SourceMgr, Diags, Tokens);
 }
 
-int doFullLexRoundTrip(const StringRef InputFilename) {
-  LangOptions LangOpts;
-  SourceManager SourceMgr;
-  DiagnosticEngine Diags(SourceMgr);
-  PrintingDiagnosticConsumer DiagPrinter;
-  Diags.addConsumer(DiagPrinter);
-
-  std::vector<std::pair<RC<syntax::TokenSyntax>,
-                                   syntax::AbsolutePosition>> Tokens;
-  if (getTokensFromFile(InputFilename, LangOpts, SourceMgr,
-                        Diags, Tokens) == EXIT_FAILURE) {
-    return EXIT_FAILURE;
-  }
-
-  for (auto TokAndPos : Tokens) {
-    TokAndPos.first->print(llvm::outs());
-  }
-
-  return Diags.hadAnyError() ? EXIT_FAILURE : EXIT_SUCCESS;
-}
-
-int doDumpTokenSyntax(const StringRef InputFilename) {
-  LangOptions LangOpts;
-  SourceManager SourceMgr;
-  DiagnosticEngine Diags(SourceMgr);
-  PrintingDiagnosticConsumer DiagPrinter;
-  Diags.addConsumer(DiagPrinter);
-
-
-  std::vector<std::pair<RC<syntax::TokenSyntax>,
-                        syntax::AbsolutePosition>> Tokens;
-  if (getTokensFromFile(InputFilename, LangOpts, SourceMgr,
-                        Diags, Tokens) == EXIT_FAILURE) {
-    return EXIT_FAILURE;
-  }
-
-  for (auto TokAndPos : Tokens) {
-    TokAndPos.second.printLineAndColumn(llvm::outs());
-    llvm::outs() << "\n";
-    TokAndPos.first->dump(llvm::outs());
-    llvm::outs() << "\n";
-  }
-
-  return Diags.hadAnyError() ? EXIT_FAILURE : EXIT_SUCCESS;
-}
-
-int doFullParseRoundTrip(const StringRef InputFilename) {
+int getSyntaxTree(const StringRef InputFilename,
+                  llvm::SmallVectorImpl<syntax::Syntax> &TopLevelDecls,
+                  std::vector<std::pair<RC<syntax::TokenSyntax>,
+                              syntax::AbsolutePosition>> &Tokens) {
   CompilerInvocation Invocation;
   Invocation.addInputFilename(InputFilename);
   Invocation.setModuleName("Test");
@@ -172,8 +135,6 @@ int doFullParseRoundTrip(const StringRef InputFilename) {
   assert(SF && "No source file");
 
   // Retokenize the buffer with full fidelity
-  std::vector<std::pair<RC<syntax::TokenSyntax>,
-                        syntax::AbsolutePosition>> Tokens;
   if (getTokensFromFile(BufferID, Invocation.getLangOptions(),
                         SourceMgr,
                         Instance.getDiags(), Tokens) == EXIT_FAILURE) {
@@ -192,11 +153,71 @@ int doFullParseRoundTrip(const StringRef InputFilename) {
     auto NewNode = transformAST(ASTNode(Decl), Sema, SourceMgr,
                                 BufferID, Tokens);
     if (NewNode.hasValue()) {
-      NewNode.getValue().print(llvm::outs());
-      auto Symbol = Sema.getNodeForSyntax(NewNode.getValue());
+      TopLevelDecls.push_back(NewNode.getValue());
     }
   }
 
+  return EXIT_SUCCESS;
+}
+
+int doFullLexRoundTrip(const StringRef InputFilename) {
+  LangOptions LangOpts;
+  SourceManager SourceMgr;
+  DiagnosticEngine Diags(SourceMgr);
+  PrintingDiagnosticConsumer DiagPrinter;
+  Diags.addConsumer(DiagPrinter);
+
+  std::vector<std::pair<RC<syntax::TokenSyntax>,
+                                   syntax::AbsolutePosition>> Tokens;
+  if (getTokensFromFile(InputFilename, LangOpts, SourceMgr,
+                        Diags, Tokens) == EXIT_FAILURE) {
+    return EXIT_FAILURE;
+  }
+
+  for (auto TokAndPos : Tokens) {
+    TokAndPos.first->print(llvm::outs());
+  }
+
+  return Diags.hadAnyError() ? EXIT_FAILURE : EXIT_SUCCESS;
+}
+
+int doDumpTokenSyntax(const StringRef InputFilename) {
+  LangOptions LangOpts;
+  SourceManager SourceMgr;
+  DiagnosticEngine Diags(SourceMgr);
+  PrintingDiagnosticConsumer DiagPrinter;
+  Diags.addConsumer(DiagPrinter);
+
+
+  std::vector<std::pair<RC<syntax::TokenSyntax>,
+  syntax::AbsolutePosition>> Tokens;
+  if (getTokensFromFile(InputFilename, LangOpts, SourceMgr,
+                        Diags, Tokens) == EXIT_FAILURE) {
+    return EXIT_FAILURE;
+  }
+
+  for (auto TokAndPos : Tokens) {
+    TokAndPos.second.printLineAndColumn(llvm::outs());
+    llvm::outs() << "\n";
+    TokAndPos.first->dump(llvm::outs());
+    llvm::outs() << "\n";
+  }
+
+  return Diags.hadAnyError() ? EXIT_FAILURE : EXIT_SUCCESS;
+}
+
+int doFullParseRoundTrip(const StringRef InputFilename) {
+
+  llvm::SmallVector<syntax::Syntax, 10> TopLevelDecls;
+  std::vector<std::pair<RC<syntax::TokenSyntax>,
+              syntax::AbsolutePosition>> Tokens;
+
+  getSyntaxTree(InputFilename, TopLevelDecls, Tokens);
+
+  for (auto &Node : TopLevelDecls) {
+    Node.print(llvm::outs());
+  }
+
   if (Tokens.back().first->getTokenKind() == tok::eof) {
     Tokens.back().first->print(llvm::outs());
   }
@@ -204,6 +225,23 @@ int doFullParseRoundTrip(const StringRef InputFilename) {
   return EXIT_SUCCESS;
 }
 
+int doSerializeRawTree(const StringRef InputFilename) {
+
+  llvm::SmallVector<syntax::Syntax, 10> TopLevelDecls;
+  std::vector<std::pair<RC<syntax::TokenSyntax>,
+  syntax::AbsolutePosition>> Tokens;
+
+  getSyntaxTree(InputFilename, TopLevelDecls, Tokens);
+
+  for (auto &Node : TopLevelDecls) {
+    swift::json::Output out(llvm::outs());
+    auto Raw = Node.getRaw();
+    out << Raw;
+  }
+
+  return EXIT_SUCCESS;
+}
+
 int main(int argc, char *argv[]) {
   llvm::cl::ParseCommandLineOptions(argc, argv, "Swift Syntax Test\n");
 
@@ -229,6 +267,9 @@ int main(int argc, char *argv[]) {
   case ActionType::FullParseRoundTrip:
     ExitCode = doFullParseRoundTrip(options::InputSourceFilename);
     break;
+  case ActionType::SerializeRawTree:
+    ExitCode = doSerializeRawTree(options::InputSourceFilename);
+    break;
   case ActionType::None:
     llvm::errs() << "an action is required\n";
     llvm::cl::PrintHelpMessage();