libSyntax: create a basic infrastructure for generating libSyntax entities by using Parser.

include/swift/AST/Module.h

@@ -28,6 +28,8 @@
 #include "swift/Basic/SourceLoc.h"
 #include "swift/Basic/STLExtras.h"
 #include "swift/Parse/Token.h"
+#include "swift/Syntax/SyntaxNodes.h"
+#include "swift/Syntax/SyntaxParsingContext.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/SetVector.h"
@@ -1082,9 +1084,23 @@ public:
     return (bool)AllCorrectedTokens;
   }
 
+  syntax::SourceFileSyntax getSyntaxRoot() const {
+    assert(SyntaxRoot && "no syntax root is set.");
+    return *SyntaxRoot;
+  }
+
 private:
+  friend class syntax::SyntaxParsingContext;
+  friend class syntax::SyntaxParsingContextRoot;
+
   /// If not None, the underlying vector should contain tokens of this source file.
   Optional<std::vector<Token>> AllCorrectedTokens;
+
+  /// All of the raw token syntax nodes in the underlying source.
+  std::vector<syntax::RawTokenInfo> AllRawTokenSyntax;
+
+  /// The root of the syntax tree representing the source file.
+  Optional<syntax::SourceFileSyntax> SyntaxRoot;
 };
 
 

include/swift/Basic/OwnedString.h

@@ -49,10 +49,6 @@ class OwnedString {
     assert(Length >= 0 && "expected length to be non-negative");
 
     if (Ownership == StringOwnership::Copied && Data) {
-      assert(
-        Length <= strlen(Data) &&
-        "expected length to be a valid index, within the length of the string");
-
       char *substring = static_cast<char *>(malloc(Length + 1));
       assert(substring && "expected successful malloc of copy");
 

include/swift/Parse/Lexer.h

@@ -23,6 +23,7 @@
 #include "swift/Parse/Token.h"
 #include "swift/Syntax/References.h"
 #include "swift/Syntax/Trivia.h"
+#include "swift/Syntax/SyntaxParsingContext.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/SaveAndRestore.h"
 
@@ -245,7 +246,7 @@ public:
   }
 
   /// Lex a full token including leading and trailing trivia.
-  RC<syntax::RawTokenSyntax> fullLex();
+  syntax::RawTokenInfo fullLex();
 
   bool isKeepingComments() const {
     return RetainComments == CommentRetentionMode::ReturnAsTokens;

include/swift/Parse/Parser.h

@@ -25,6 +25,8 @@
 #include "swift/AST/Module.h"
 #include "swift/AST/Pattern.h"
 #include "swift/AST/Stmt.h"
+#include "swift/Syntax/RawTokenSyntax.h"
+#include "swift/Syntax/SyntaxParsingContext.h"
 #include "swift/Basic/OptionSet.h"
 #include "swift/Parse/Lexer.h"
 #include "swift/Parse/LocalContext.h"
@@ -325,6 +327,9 @@ public:
   /// This vector is managed by \c StructureMarkerRAII objects.
   llvm::SmallVector<StructureMarker, 16> StructureMarkers;
 
+  /// Current syntax parsing context where call backs should be directed to.
+  syntax::SyntaxParsingContext *SyntaxContext;
+
 public:
   Parser(unsigned BufferID, SourceFile &SF, SILParserTUStateBase *SIL,
          PersistentParserState *PersistentState = nullptr);
@@ -1417,6 +1422,11 @@ tokenizeWithTrivia(const LangOptions &LangOpts,
                    unsigned Offset = 0,
                    unsigned EndOffset = 0);
 
+
+void populateTokenSyntaxMap(const LangOptions &LangOpts,
+                            const SourceManager &SM,
+                            unsigned BufferID,
+                            std::vector<syntax::RawTokenInfo> &Result);
 } // end namespace swift
 
 #endif

include/swift/Syntax/Syntax.h

@@ -127,6 +127,9 @@ public:
   /// if it has one, otherwise 0.
   CursorIndex getIndexInParent() const;
 
+  /// Returns true if this syntax node represents a token.
+  bool isToken() const;
+
   /// Returns true if this syntax node represents a statement.
   bool isStmt() const;
 
@@ -152,9 +155,6 @@ public:
   /// Returns true if the node is "present" in the source.
   bool isPresent() const;
 
-
-  bool isToken() const;
-
   /// Print the syntax node with full fidelity to the given output stream.
   void print(llvm::raw_ostream &OS) const;
 

include/swift/Syntax/SyntaxFactory.h.gyb

@@ -54,6 +54,8 @@ struct SyntaxFactory {
   static Optional<Syntax>
   createSyntax(SyntaxKind Kind, llvm::ArrayRef<Syntax> Elements);
 
+  static SyntaxKind getUnknownKind(SyntaxKind Kind);
+
 % for node in SYNTAX_NODES:
 %   if node.children:
 %     child_params = []

include/swift/Syntax/SyntaxParsingContext.h

@@ -0,0 +1,110 @@
+//===----------- SyntaxParsingContext.h -==============----------*- C++ -*-===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SWIFT_SYNTAX_PARSING_CONTEXT_H
+#define SWIFT_SYNTAX_PARSING_CONTEXT_H
+
+#include "swift/Syntax/RawTokenSyntax.h"
+#include "swift/Syntax/TokenSyntax.h"
+#include "swift/Syntax/References.h"
+#include "swift/Syntax/RawSyntax.h"
+#include "swift/Syntax/Syntax.h"
+#include "swift/Syntax/TokenKinds.h"
+#include "swift/Syntax/Trivia.h"
+
+namespace swift {
+  class SourceFile;
+
+namespace syntax {
+
+struct RawTokenInfo {
+  SourceLoc Loc;
+  RC<RawTokenSyntax> Token;
+};
+
+enum class SyntaxParsingContextKind: uint8_t {
+  Root,
+  Child,
+};
+
+/// The base class of different kinds of Syntax context that Parser should use to
+/// create syntax nodes.
+class SyntaxParsingContext {
+protected:
+  SyntaxParsingContext(bool Enabled);
+  SyntaxParsingContext(SyntaxParsingContext &Another);
+public:
+  struct ContextInfo;
+  ContextInfo &ContextData;
+
+  // Add a token syntax at the given source location to the context; this
+  // token node can be used to build more complex syntax nodes in later call
+  // back.
+  virtual void addTokenSyntax(SourceLoc Loc) = 0;
+
+  // Get the context kind.
+  virtual SyntaxParsingContextKind getKind() = 0;
+
+  // Create a syntax node of the given kind.
+  virtual void makeNode(SyntaxKind Kind) = 0;
+  virtual ~SyntaxParsingContext();
+
+  // Disable the building of syntax tree in the current context.
+  void disable();
+};
+
+// The start point of syntax tree parsing. This context is the root
+// of all other entity-specific contexts. This is the context Parser
+// has when the parser instance is firstly created.
+class SyntaxParsingContextRoot: public SyntaxParsingContext {
+public:
+  struct GlobalInfo;
+
+  // Contains global information of the source file under parsing.
+  GlobalInfo &GlobalData;
+  SyntaxParsingContextRoot(SourceFile &SF, unsigned BufferID);
+  ~SyntaxParsingContextRoot();
+  void addTokenSyntax(SourceLoc Loc) override {};
+  void makeNode(SyntaxKind Kind) override {};
+  SyntaxParsingContextKind getKind() override {
+    return SyntaxParsingContextKind::Root;
+  };
+};
+
+// The base class for contexts that are created from a parent context.
+// The stack instance will set the context holder when the context
+// is firstly created and reset the context holder to the parent when
+// it's destructed.
+class SyntaxParsingContextChild: public SyntaxParsingContext {
+  SyntaxParsingContext *Parent;
+  SyntaxParsingContext *&ContextHolder;
+  const SyntaxKind FinalKind;
+public:
+  SyntaxParsingContextChild(SyntaxParsingContext *&ContextHolder,
+                            SyntaxKind FinalKind):
+    SyntaxParsingContext(*ContextHolder), Parent(ContextHolder),
+    ContextHolder(ContextHolder), FinalKind(FinalKind) {
+      ContextHolder = this;
+  }
+  ~SyntaxParsingContextChild();
+  void makeNode(SyntaxKind Kind) override;
+  void addTokenSyntax(SourceLoc Loc) override;
+  SyntaxParsingContext* getParent() { return Parent; }
+  SyntaxParsingContextRoot &getRoot();
+  SyntaxParsingContextKind getKind() override {
+    return SyntaxParsingContextKind::Child;
+  };
+};
+}
+}
+#endif // SWIFT_SYNTAX_PARSING_CONTEXT_H
+

lib/Parse/Lexer.cpp

@@ -20,7 +20,7 @@
 #include "swift/AST/Identifier.h"
 #include "swift/Basic/LangOptions.h"
 #include "swift/Basic/SourceManager.h"
-#include "swift/Syntax/TokenSyntax.h"
+#include "swift/Syntax/SyntaxParsingContext.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/MemoryBuffer.h"
@@ -737,11 +737,12 @@ static bool rangeContainsPlaceholderEnd(const char *CurPtr,
   return false;
 }
 
-RC<syntax::RawTokenSyntax> Lexer::fullLex() {
+syntax::RawTokenInfo Lexer::fullLex() {
   if (NextToken.isEscapedIdentifier()) {
     LeadingTrivia.push_back(syntax::TriviaPiece::backtick());
     TrailingTrivia.push_front(syntax::TriviaPiece::backtick());
   }
+  auto Loc = NextToken.getLoc();
   auto Result = syntax::RawTokenSyntax::make(NextToken.getKind(),
                                         OwnedString(NextToken.getText()).copy(),
                                         syntax::SourcePresence::Present,
@@ -751,7 +752,7 @@ RC<syntax::RawTokenSyntax> Lexer::fullLex() {
   if (NextToken.isNot(tok::eof)) {
     lexImpl();
   }
-  return Result;
+  return {Loc, Result};
 }
 
 /// lexOperatorIdentifier - Match identifiers formed out of punctuation.

lib/Parse/ParseExpr.cpp

@@ -18,6 +18,9 @@
 #include "swift/AST/DiagnosticsParse.h"
 #include "swift/Basic/EditorPlaceholder.h"
 #include "swift/Parse/CodeCompletionCallbacks.h"
+#include "swift/Syntax/SyntaxFactory.h"
+#include "swift/Syntax/TokenSyntax.h"
+#include "swift/Syntax/SyntaxParsingContext.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/ADT/Twine.h"
@@ -27,6 +30,7 @@
 #include "llvm/Support/raw_ostream.h"
 
 using namespace swift;
+using namespace swift::syntax;
 
 /// parseExpr
 ///
@@ -35,6 +39,9 @@ using namespace swift;
 ///
 /// \param isExprBasic Whether we're only parsing an expr-basic.
 ParserResult<Expr> Parser::parseExprImpl(Diag<> Message, bool isExprBasic) {
+  // Start a context for creating expression syntax.
+  SyntaxParsingContextChild ExprParsingContext(SyntaxContext, SyntaxKind::Expr);
+
   // If we are parsing a refutable pattern, check to see if this is the start
   // of a let/var/is pattern.  If so, parse it to an UnresolvedPatternExpr and
   // name binding will perform final validation.
@@ -465,6 +472,7 @@ ParserResult<Expr> Parser::parseExprUnary(Diag<> Message, bool isExprBasic) {
     Tok.setKind(tok::oper_prefix);
     LLVM_FALLTHROUGH;
   case tok::oper_prefix:
+    SyntaxContext->addTokenSyntax(Tok.getLoc());
     Operator = parseExprOperator();
     break;
   case tok::oper_binary_spaced:
@@ -757,7 +765,6 @@ UnresolvedDeclRefExpr *Parser::parseExprOperator() {
   SourceLoc loc = Tok.getLoc();
   Identifier name = Context.getIdentifier(Tok.getText());
   consumeToken();
-
   // Bypass local lookup.
   return new (Context) UnresolvedDeclRefExpr(name, refKind, DeclNameLoc(loc));
 }
@@ -1381,6 +1388,8 @@ ParserResult<Expr> Parser::parseExprPostfix(Diag<> ID, bool isExprBasic) {
   case tok::integer_literal: {
     StringRef Text = copyAndStripUnderscores(Context, Tok.getText());
     SourceLoc Loc = consumeToken(tok::integer_literal);
+    SyntaxContext->addTokenSyntax(Loc);
+    SyntaxContext->makeNode(SyntaxKind::IntegerLiteralExpr);
     Result = makeParserResult(new (Context) IntegerLiteralExpr(Text, Loc,
                                                            /*Implicit=*/false));
     break;
@@ -1786,11 +1795,20 @@ createStringLiteralExprFromSegment(ASTContext &Ctx,
 ///   expr-literal:
 ///     string_literal
 ParserResult<Expr> Parser::parseExprStringLiteral() {
+
   SmallVector<Lexer::StringSegment, 1> Segments;
   L->getStringLiteralSegments(Tok, Segments);
 
   Token EntireTok = Tok;
 
+  // Create a syntax node for string literal.
+  SyntaxContext->addTokenSyntax(Tok.getLoc());
+  SyntaxContext->makeNode(SyntaxKind::StringLiteralExpr);
+  SyntaxParsingContextChild LocalContext(SyntaxContext, SyntaxKind::Expr);
+
+  // FIXME: Avoid creating syntax nodes for string interpolation.
+  LocalContext.disable();
+
   // The start location of the entire string literal.
   SourceLoc Loc = Tok.getLoc();
 

lib/Parse/Parser.cpp

@@ -268,7 +268,7 @@ swift::tokenizeWithTrivia(const LangOptions &LangOpts,
                         syntax::AbsolutePosition>> Tokens;
   syntax::AbsolutePosition RunningPos;
   do {
-    auto ThisToken = L.fullLex();
+    auto ThisToken = L.fullLex().Token;
     auto ThisTokenPos = ThisToken->accumulateAbsolutePosition(RunningPos);
     Tokens.push_back({ThisToken, ThisTokenPos});
   } while (Tokens.back().first->isNot(tok::eof));
@@ -276,6 +276,22 @@ swift::tokenizeWithTrivia(const LangOptions &LangOpts,
   return Tokens;
 }
 
+void swift::populateTokenSyntaxMap(const LangOptions &LangOpts,
+                                   const SourceManager &SM,
+                                   unsigned BufferID,
+                                   std::vector<syntax::RawTokenInfo> &Result) {
+  if (!Result.empty())
+    return;
+  Lexer L(LangOpts, SM, BufferID, /*Diags=*/nullptr, /*InSILMode=*/false,
+          CommentRetentionMode::AttachToNextToken,
+          TriviaRetentionMode::WithTrivia);
+  do {
+    Result.emplace_back(L.fullLex());
+    if (Result.back().Token->is(tok::eof))
+      return;
+  } while (true);
+}
+
 //===----------------------------------------------------------------------===//
 // Setup and Helper Methods
 //===----------------------------------------------------------------------===//
@@ -417,7 +433,8 @@ Parser::Parser(std::unique_ptr<Lexer> Lex, SourceFile &SF,
     Context(SF.getASTContext()),
     TokReceiver(SF.shouldKeepTokens() ?
                 new TokenRecorder(SF) :
-                new ConsumeTokenReceiver())  {
+                new ConsumeTokenReceiver()),
+    SyntaxContext(new syntax::SyntaxParsingContextRoot(SF, L->getBufferID())) {
 
   State = PersistentState;
   if (!State) {
@@ -441,6 +458,7 @@ Parser::Parser(std::unique_ptr<Lexer> Lex, SourceFile &SF,
 Parser::~Parser() {
   delete L;
   delete TokReceiver;
+  delete SyntaxContext;
 }
 
 const Token &Parser::peekToken() {

lib/Syntax/CMakeLists.txt

@@ -12,5 +12,6 @@ add_swift_library(swiftSyntax STATIC
   Syntax.cpp
   SyntaxData.cpp
   UnknownSyntax.cpp
+  SyntaxParsingContext.cpp
   DEPENDS
     swift-syntax-generated-headers)

lib/Syntax/Syntax.cpp

@@ -52,6 +52,10 @@ bool Syntax::isExpr() const {
   return Data->isExpr();
 }
 
+bool Syntax::isToken() const {
+  return getRaw()->isToken();
+}
+
 bool Syntax::isPattern() const {
   return Data->isPattern();
 }
@@ -68,11 +72,6 @@ bool Syntax::isMissing() const {
   return getRaw()->isMissing();
 }
 
-bool Syntax::isToken() const {
-  return getRaw()->isToken();
-}
-
-
 llvm::Optional<Syntax> Syntax::getParent() const {
   auto ParentData = getData().Parent;
   if (ParentData == nullptr) return llvm::None;

lib/Syntax/SyntaxFactory.cpp.gyb

@@ -160,6 +160,28 @@ SyntaxFactory::createSyntax(SyntaxKind Kind, llvm::ArrayRef<Syntax> Elements) {
   }
 }
 
+SyntaxKind
+SyntaxFactory::getUnknownKind(SyntaxKind Kind) {
+  switch(Kind) {
+% for node in SYNTAX_NODES:
+%   if node.syntax_kind.endswith('Expr'):
+%     Result = 'SyntaxKind::UnknownExpr'
+%   elif node.syntax_kind.endswith('Stmt'):
+%     Result = 'SyntaxKind::UnknownStmt'
+%   elif node.syntax_kind.endswith('Decl'):
+%     Result = 'SyntaxKind::UnknownDecl'
+%   elif node.syntax_kind.endswith('Token'):
+%     Result = 'SyntaxKind::UnknownToken'
+%   else:
+%     Result = 'SyntaxKind::Unknown'
+%   end
+  case SyntaxKind::${node.syntax_kind}: return ${Result};
+% end
+  default:
+    return SyntaxKind::Unknown;
+  }
+}
+
 % for node in SYNTAX_NODES:
 %   if node.children:
 %     child_params = []

lib/Syntax/SyntaxParsingContext.cpp

@@ -0,0 +1,211 @@
+//===--- SyntaxParsingContext.cpp - Syntax Tree Parsing Support------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+#include "swift/AST/Module.h"
+#include "swift/Basic/Defer.h"
+#include "swift/Parse/Parser.h"
+#include "swift/Syntax/TokenSyntax.h"
+#include "swift/Syntax/SyntaxParsingContext.h"
+#include "swift/Syntax/SyntaxFactory.h"
+
+using namespace swift;
+using namespace swift::syntax;
+
+namespace {
+static Syntax makeUnknownSyntax(SyntaxKind Kind, ArrayRef<Syntax> SubExpr) {
+  RawSyntax::LayoutList Layout;
+  std::transform(SubExpr.begin(), SubExpr.end(), std::back_inserter(Layout),
+                 [](const Syntax &S) { return S.getRaw(); });
+  return make<Syntax>(RawSyntax::make(Kind, Layout, SourcePresence::Present));
+}
+} // End of anonymous namespace
+
+struct SyntaxParsingContext::ContextInfo {
+  bool Enabled;
+  std::vector<Syntax> PendingSyntax;
+
+  ContextInfo(bool Enabled): Enabled(Enabled) {}
+
+  // Check if the pending syntax is a token syntax in the given kind.
+  bool checkTokenFromBack(tok Kind, unsigned OffsetFromBack = 0) {
+    if (PendingSyntax.size() - 1 < OffsetFromBack)
+      return false;
+    auto Back = PendingSyntax[PendingSyntax.size() - 1 - OffsetFromBack].
+      getAs<TokenSyntax>();
+    return Back.hasValue() && Back->getTokenKind() == Kind;
+  }
+
+  void addPendingSyntax(ArrayRef<Syntax> More) {
+    std::transform(More.begin(), More.end(), std::back_inserter(PendingSyntax),
+                 [](const Syntax &S) { return make<Syntax>(S.getRaw()); });
+  }
+
+  // Squash N syntax nodex from the back of the pending list into one.
+  void createFromBack(SyntaxKind Kind, unsigned N = 0) {
+    auto Size = PendingSyntax.size();
+    assert(Size >= N);
+    if (!N)
+      N = Size;
+    auto Parts = llvm::makeArrayRef(PendingSyntax).slice(Size - N);
+
+    // Try to create the node of the given syntax.
+    Optional<Syntax> Result = SyntaxFactory::createSyntax(Kind, Parts);
+    if (!Result) {
+
+      // If unable to create, we should create an unknown node.
+      Result.emplace(makeUnknownSyntax(SyntaxFactory::getUnknownKind(Kind),
+                                       Parts));
+    }
+
+    // Remove the building bricks and re-append the result.
+    for (unsigned I = 0; I < N; I ++)
+      PendingSyntax.pop_back();
+    addPendingSyntax({ *Result });
+    assert(Size - N + 1 == PendingSyntax.size());
+  }
+};
+
+SyntaxParsingContext::SyntaxParsingContext(bool Enabled):
+  ContextData(*new ContextInfo(Enabled)) {}
+
+SyntaxParsingContext::SyntaxParsingContext(SyntaxParsingContext &Another):
+    SyntaxParsingContext(Another.ContextData.Enabled) {}
+
+SyntaxParsingContext::~SyntaxParsingContext() { delete &ContextData; }
+
+void SyntaxParsingContext::disable() { ContextData.Enabled = false; }
+
+struct SyntaxParsingContextRoot::GlobalInfo {
+  // The source file under parsing.
+  SourceFile &File;
+
+  // All tokens in the source file. This list will shrink from the start when
+  // we start to build syntax nodes.
+  ArrayRef<RawTokenInfo> Tokens;
+
+  GlobalInfo(SourceFile &File) : File(File) {}
+  TokenSyntax retrieveTokenSyntax(SourceLoc Loc) {
+    auto TargetLoc = Loc.getOpaquePointerValue();
+    for (unsigned I = 0, N = Tokens.size(); I < N; I ++) {
+      auto Info = Tokens[I];
+      auto InfoLoc = Info.Loc.getOpaquePointerValue();
+      if (InfoLoc < TargetLoc)
+        continue;
+      assert(InfoLoc == TargetLoc);
+      Tokens = Tokens.slice(I + 1);
+      return make<TokenSyntax>(Info.Token);
+    }
+    llvm_unreachable("can not find token at Loc");
+  }
+};
+
+SyntaxParsingContextRoot::
+SyntaxParsingContextRoot(SourceFile &File, unsigned BufferID):
+    SyntaxParsingContext(File.shouldKeepTokens()),
+    GlobalData(*new GlobalInfo(File)) {
+  populateTokenSyntaxMap(File.getASTContext().LangOpts,
+                         File.getASTContext().SourceMgr,
+                         BufferID, File.AllRawTokenSyntax);
+  // Keep track of the raw tokens.
+  GlobalData.Tokens = llvm::makeArrayRef(File.AllRawTokenSyntax);
+}
+
+SyntaxParsingContextRoot::~SyntaxParsingContextRoot() {
+  std::vector<DeclSyntax> AllTopLevel;
+  if (GlobalData.File.SyntaxRoot.hasValue()) {
+    for (auto It: GlobalData.File.getSyntaxRoot().getTopLevelDecls()) {
+      AllTopLevel.push_back(It);
+    }
+  }
+  for (auto S: ContextData.PendingSyntax) {
+    std::vector<StmtSyntax> AllStmts;
+    if (S.isDecl()) {
+      AllStmts.push_back(SyntaxFactory::makeDeclarationStmt(
+        S.getAs<DeclSyntax>().getValue(), None));
+    } else if (S.isExpr()) {
+      AllStmts.push_back(SyntaxFactory::makeExpressionStmt(
+        S.getAs<ExprSyntax>().getValue(), None));
+    } else if (S.isStmt()) {
+      AllStmts.push_back(S.getAs<StmtSyntax>().getValue());
+    } else {
+      // If this is a standalone token, we create an unknown expression wrapper
+      // for it.
+      AllStmts.push_back(SyntaxFactory::makeExpressionStmt(
+        *makeUnknownSyntax(SyntaxKind::UnknownExpr,
+                           { *S.getAs<TokenSyntax>() }).getAs<ExprSyntax>(),
+                                                           None));
+    }
+    AllTopLevel.push_back(SyntaxFactory::makeTopLevelCodeDecl(
+      SyntaxFactory::makeStmtList(AllStmts)));
+  }
+
+  Trivia Leading = Trivia::newlines(1), Trailing;
+  GlobalData.File.SyntaxRoot.emplace(
+    SyntaxFactory::makeSourceFile(SyntaxFactory::makeDeclList(AllTopLevel),
+      SyntaxFactory::makeToken(tok::eof, "", SourcePresence::Present,
+                               Leading, Trailing)));
+  delete &GlobalData;
+}
+
+SyntaxParsingContextRoot &SyntaxParsingContextChild::getRoot() {
+  for (SyntaxParsingContext *Root = getParent(); ;
+       Root = static_cast<SyntaxParsingContextChild*>(Root)->getParent()){
+    if (Root->getKind() == SyntaxParsingContextKind::Root)
+      return *static_cast<SyntaxParsingContextRoot*>(Root);
+  }
+  llvm_unreachable("can not find root");
+}
+
+void SyntaxParsingContextChild::addTokenSyntax(SourceLoc Loc) {
+  if (ContextData.Enabled)
+    ContextData.PendingSyntax.push_back(getRoot().GlobalData.
+      retrieveTokenSyntax(Loc));
+}
+
+void SyntaxParsingContextChild::makeNode(SyntaxKind Kind) {
+  if (!ContextData.Enabled)
+    return;
+
+  // Create syntax nodes according to the given kind.
+  switch (Kind) {
+  case SyntaxKind::IntegerLiteralExpr: {
+    // Integer may include the signs before the digits, so check if the sign
+    // exists and create.
+    ContextData.createFromBack(Kind, ContextData.
+      checkTokenFromBack(tok::oper_prefix, 1) ? 2 : 1);
+    break;
+  }
+  case SyntaxKind::StringLiteralExpr: {
+    ContextData.createFromBack(Kind, 1);
+    break;
+  }
+
+  default:
+    break;
+  }
+}
+
+SyntaxParsingContextChild::~SyntaxParsingContextChild() {
+  SWIFT_DEFER {
+    // Parent should take care of the created syntax.
+    Parent->ContextData.addPendingSyntax(ContextData.PendingSyntax);
+
+    // Reset the context holder to be Parent.
+    ContextHolder = Parent;
+  };
+
+  // If we've created more than one syntax node, we should try building by using
+  // the final kind.
+  if (ContextData.PendingSyntax.size() > 1) {
+    ContextData.createFromBack(FinalKind);
+  }
+}

test/Syntax/round_trip_parse_gen.swift

@@ -0,0 +1,8 @@
+// RUN: %swift-syntax-test -input-source-filename %s -parse-gen > %t
+// RUN: diff %t %s
+
+"String Literal"
+
+""
+
+/*comments*/+3 // comments

tools/swift-syntax-test/swift-syntax-test.cpp

@@ -42,6 +42,7 @@ enum class ActionType {
   FullLexRoundTrip,
   FullParseRoundTrip,
   SerializeRawTree,
+  ParserGen,
   None
 };
 
@@ -62,6 +63,10 @@ Action(llvm::cl::desc("Action (required):"),
                    "round-trip-parse",
                    "Parse the source file and print it back out for "
                    "comparing against the input"),
+        clEnumValN(ActionType::ParserGen,
+                   "parse-gen",
+                   "Parse the source file and print it back out for "
+                   "comparing against the input"),
         clEnumValN(ActionType::SerializeRawTree,
                    "serialize-raw-tree",
                    "Parse the source file and serialize the raw tree"
@@ -72,6 +77,7 @@ InputSourceFilename("input-source-filename",
                     llvm::cl::desc("Path to the input .swift file"));
 } // end namespace options
 
+namespace {
 int getTokensFromFile(unsigned BufferID,
                       LangOptions &LangOpts,
                       SourceManager &SourceMgr,
@@ -103,37 +109,27 @@ getTokensFromFile(const StringRef InputFilename,
 
 void anchorForGetMainExecutable() {}
 
-int getSyntaxTree(const char *MainExecutablePath,
-                  const StringRef InputFilename,
-                  CompilerInstance &Instance,
-                  llvm::SmallVectorImpl<syntax::Syntax> &TopLevelDecls,
-                  std::vector<std::pair<RC<syntax::RawTokenSyntax>,
-                              syntax::AbsolutePosition>> &Tokens) {
-  CompilerInvocation Invocation;
-  Invocation.addInputFilename(InputFilename);
 
+SourceFile *getSourceFile(CompilerInstance &Instance,
+                          StringRef InputFileName,
+                          const char *MainExecutablePath) {
+  CompilerInvocation Invocation;
+  Invocation.getLangOptions().KeepTokensInSourceFile = true;
+  Invocation.addInputFilename(InputFileName);
   Invocation.setMainExecutablePath(
     llvm::sys::fs::getMainExecutable(MainExecutablePath,
       reinterpret_cast<void *>(&anchorForGetMainExecutable)));
 
   Invocation.setModuleName("Test");
-
-  auto &SourceMgr = Instance.getSourceMgr();
-
   PrintingDiagnosticConsumer DiagPrinter;
   Instance.addDiagnosticConsumer(&DiagPrinter);
   if (Instance.setup(Invocation)) {
-    return EXIT_FAILURE;
+    return nullptr;
   }
 
   // First, parse the file normally and get the regular old AST.
   Instance.performParseOnly();
 
-  if (Instance.getDiags().hadAnyError()) {
-    return EXIT_FAILURE;
-  }
-
-  auto BufferID = Instance.getInputBufferIDs().back();
   SourceFile *SF = nullptr;
   for (auto Unit : Instance.getMainModule()->getFiles()) {
     SF = dyn_cast<SourceFile>(Unit);
@@ -142,9 +138,21 @@ int getSyntaxTree(const char *MainExecutablePath,
     }
   }
   assert(SF && "No source file");
+  return SF;
+}
+
+int getSyntaxTree(const char *MainExecutablePath,
+                  const StringRef InputFilename,
+                  CompilerInstance &Instance,
+                  llvm::SmallVectorImpl<syntax::Syntax> &TopLevelDecls,
+                  std::vector<std::pair<RC<syntax::RawTokenSyntax>,
+                              syntax::AbsolutePosition>> &Tokens) {
+  auto *SF = getSourceFile(Instance, InputFilename, MainExecutablePath);
+  auto &SourceMgr = Instance.getSourceMgr();
+  auto BufferID = Instance.getInputBufferIDs().back();
 
   // Retokenize the buffer with full fidelity
-  if (getTokensFromFile(BufferID, Invocation.getLangOptions(),
+  if (getTokensFromFile(BufferID, SF->getASTContext().LangOpts,
                         SourceMgr,
                         Instance.getDiags(), Tokens) == EXIT_FAILURE) {
     return EXIT_FAILURE;
@@ -260,6 +268,16 @@ int doSerializeRawTree(const char *MainExecutablePath,
   return EXIT_SUCCESS;
 }
 
+int dumpParserGen(const char *MainExecutablePath,
+                  const StringRef InputFileName) {
+  CompilerInstance Instance;
+  SourceFile *SF = getSourceFile(Instance, InputFileName, MainExecutablePath);
+  SF->getSyntaxRoot().print(llvm::outs());
+  return 0;
+}
+
+}// end of anonymous namespace
+
 int main(int argc, char *argv[]) {
   llvm::cl::ParseCommandLineOptions(argc, argv, "Swift Syntax Test\n");
 
@@ -288,6 +306,9 @@ int main(int argc, char *argv[]) {
   case ActionType::SerializeRawTree:
     ExitCode = doSerializeRawTree(argv[0], options::InputSourceFilename);
     break;
+  case ActionType::ParserGen:
+    ExitCode = dumpParserGen(argv[0], options::InputSourceFilename);
+    break;
   case ActionType::None:
     llvm::errs() << "an action is required\n";
     llvm::cl::PrintHelpMessage();