libSyntax: optionally emit diagnostics for unknown expressions and declarations. (#13973)

With more syntax nodes being specialized, we'd like this
straight-forward way to pinpoint unknown entities. This diagnostics
is only issued in -emit-syntax frontend action and swift-syntax-test
invocation.

include/swift/AST/DiagnosticsParse.def

@@ -1553,6 +1553,12 @@ ERROR(pound_available_swift_not_allowed, none,
 ERROR(availability_query_repeated_platform, none,
       "version for '%0' already specified", (StringRef))
 
+//------------------------------------------------------------------------------
+// syntax parsing diagnostics
+//------------------------------------------------------------------------------
+WARNING(unknown_syntax_entity, PointsToFirstBadToken,
+        "unknown %0 syntax exists in the source", (StringRef))
+
 #ifndef DIAG_NO_UNDEF
 # if defined(DIAG)
 #  undef DIAG

include/swift/Basic/LangOptions.h

@@ -256,6 +256,9 @@ namespace swift {
     /// this source file.
     bool KeepSyntaxInfoInSourceFile = false;
 
+    /// Whether to verify the parsed syntax tree and emit related diagnostics.
+    bool VerifySyntaxTree = false;
+
     /// Sets the target we are building for and updates platform conditions
     /// to match.
     ///

include/swift/Syntax/SyntaxParsingContext.h

@@ -73,10 +73,18 @@ struct alignas(1 << SyntaxAlignInBits) RootContextData {
   // Where to issue diagnostics.
   DiagnosticEngine &Diags;
 
+  SourceManager &SourceMgr;
+
+  unsigned BufferID;
+
   // Storage for Collected parts.
   std::vector<RC<RawSyntax>> Storage;
 
-  RootContextData(SourceFile &SF, DiagnosticEngine &Diags): SF(SF), Diags(Diags) {}
+  RootContextData(SourceFile &SF,
+                  DiagnosticEngine &Diags,
+                  SourceManager &SourceMgr,
+                  unsigned BufferID): SF(SF), Diags(Diags),
+                    SourceMgr(SourceMgr), BufferID(BufferID) {}
 };
 
 /// RAII object which receive RawSyntax parts. On destruction, this constructs
@@ -133,7 +141,7 @@ class alignas(1 << SyntaxAlignInBits) SyntaxParsingContext {
 public:
   /// Construct root context.
   SyntaxParsingContext(SyntaxParsingContext *&CtxtHolder, SourceFile &SF,
-    DiagnosticEngine &Diags);
+    DiagnosticEngine &Diags, SourceManager &SourceMgr, unsigned BufferID);
 
   /// Designated constructor for child context.
   SyntaxParsingContext(SyntaxParsingContext *&CtxtHolder)

lib/FrontendTool/FrontendTool.cpp

@@ -515,8 +515,10 @@ static bool performCompile(CompilerInstance &Instance,
   FrontendOptions opts = Invocation.getFrontendOptions();
   FrontendOptions::ActionType Action = opts.RequestedAction;
 
-  if (Action == FrontendOptions::ActionType::EmitSyntax)
+  if (Action == FrontendOptions::ActionType::EmitSyntax) {
     Instance.getASTContext().LangOpts.KeepSyntaxInfoInSourceFile = true;
+    Instance.getASTContext().LangOpts.VerifySyntaxTree = true;
+  }
 
   // We've been asked to precompile a bridging header; we want to
   // avoid touching any other inputs and just parse, emit and exit.

lib/Parse/Parser.cpp

@@ -467,7 +467,8 @@ Parser::Parser(std::unique_ptr<Lexer> Lex, SourceFile &SF,
     TokReceiver(SF.shouldKeepSyntaxInfo() ?
                 new TokenRecorder(SF) :
                 new ConsumeTokenReceiver()),
-    SyntaxContext(new SyntaxParsingContext(SyntaxContext, SF, Diags)) {
+    SyntaxContext(new SyntaxParsingContext(SyntaxContext, SF, Diags, SourceMgr,
+                                           L->getBufferID())) {
   State = PersistentState;
   if (!State) {
     OwnedState.reset(new PersistentParserState());

lib/Syntax/SyntaxParsingContext.cpp

@@ -20,6 +20,7 @@
 #include "swift/Syntax/References.h"
 #include "swift/Syntax/Syntax.h"
 #include "swift/Syntax/SyntaxFactory.h"
+#include "swift/Syntax/SyntaxVisitor.h"
 #include "swift/Syntax/TokenKinds.h"
 #include "swift/Syntax/TokenSyntax.h"
 #include "swift/Syntax/Trivia.h"
@@ -52,10 +53,12 @@ RC<RawSyntax> createSyntaxAs(SyntaxKind Kind, ArrayRef<RC<RawSyntax>> Parts) {
 
 SyntaxParsingContext::SyntaxParsingContext(SyntaxParsingContext *&CtxtHolder,
                                            SourceFile &SF,
-                                           DiagnosticEngine &Diags)
-    : RootDataOrParent(new RootContextData(SF, Diags)), CtxtHolder(CtxtHolder),
-      Storage(getRootData().Storage), Offset(0), Mode(AccumulationMode::Root),
-      Enabled(SF.shouldKeepSyntaxInfo()) {
+                                           DiagnosticEngine &Diags,
+                                           SourceManager &SourceMgr,
+                                           unsigned BufferID)
+    : RootDataOrParent(new RootContextData(SF, Diags, SourceMgr, BufferID)),
+      CtxtHolder(CtxtHolder), Storage(getRootData().Storage), Offset(0),
+      Mode(AccumulationMode::Root), Enabled(SF.shouldKeepSyntaxInfo()) {
   CtxtHolder = this;
 }
 
@@ -214,7 +217,37 @@ RC<RawSyntax> bridgeAs(SyntaxContextKind Kind, ArrayRef<RC<RawSyntax>> Parts) {
   }
 }
 
-void finalizeSourceFile(SourceFile &SF, ArrayRef<RC<RawSyntax>> Parts) {
+/// This verifier traverses a syntax node to emit proper diagnostics.
+class SyntaxVerifier: public SyntaxVisitor {
+  SourceFileSyntax Root;
+  RootContextData &RootData;
+  template<class T>
+  SourceLoc getSourceLoc(T Node) {
+    return RootData.SourceMgr.getLocForOffset(RootData.BufferID,
+      Node.getAbsolutePosition(Root).getOffset());
+  }
+public:
+  SyntaxVerifier(SourceFileSyntax Root, RootContextData &RootData) :
+    Root(Root), RootData(RootData) {}
+  void visit(UnknownDeclSyntax Node) override {
+    RootData.Diags.diagnose(getSourceLoc(Node), diag::unknown_syntax_entity,
+                            "declaration");
+    visitChildren(Node);
+  }
+  void visit(UnknownExprSyntax Node) override {
+    RootData.Diags.diagnose(getSourceLoc(Node), diag::unknown_syntax_entity,
+                            "expression");
+    visitChildren(Node);
+  }
+
+  void verify(Syntax Node) {
+    Node.accept(*this);
+  }
+};
+
+void finalizeSourceFile(RootContextData &RootData,
+                        ArrayRef<RC<RawSyntax>> Parts) {
+  SourceFile &SF = RootData.SF;
   std::vector<DeclSyntax> AllTopLevel;
   llvm::Optional<TokenSyntax> EOFToken;
 
@@ -242,6 +275,14 @@ void finalizeSourceFile(SourceFile &SF, ArrayRef<RC<RawSyntax>> Parts) {
       SyntaxFactory::makeDeclList(AllTopLevel),
       EOFToken.hasValue() ? *EOFToken
                           : TokenSyntax::missingToken(tok::eof, "")));
+
+  if (SF.getASTContext().LangOpts.VerifySyntaxTree) {
+    // Verify the added nodes if specified.
+    SyntaxVerifier Verifier(SF.getSyntaxRoot(), RootData);
+    for (auto RawNode: Parts) {
+      Verifier.verify(make<Syntax>(RawNode));
+    }
+  }
 }
 } // End of anonymous namespace
 
@@ -294,7 +335,7 @@ SyntaxParsingContext::~SyntaxParsingContext() {
   // Accumulate parsed toplevel syntax onto the SourceFile.
   case AccumulationMode::Root:
     assert(isRoot() && "AccumulationMode::Root is only for root context");
-    finalizeSourceFile(getRootData().SF, getParts());
+    finalizeSourceFile(getRootData(), getParts());
     break;
 
   // Never.

test/Syntax/syntax_diagnostics.swift

@@ -0,0 +1,3 @@
+// RUN: %target-swift-frontend -emit-syntax -primary-file %s -verify
+
+typealias Inner: Foo // expected-warning{{unknown declaration syntax exists in the source}} expected-error{{expected '=' in type alias declaration}}

tools/swift-syntax-test/swift-syntax-test.cpp

@@ -145,6 +145,7 @@ SourceFile *getSourceFile(CompilerInstance &Instance,
                           const char *MainExecutablePath) {
   CompilerInvocation Invocation;
   Invocation.getLangOptions().KeepSyntaxInfoInSourceFile = true;
+  Invocation.getLangOptions().VerifySyntaxTree = true;
   Invocation.getFrontendOptions().Inputs.addInputFile(InputFileName);
   Invocation.setMainExecutablePath(
     llvm::sys::fs::getMainExecutable(MainExecutablePath,