fix rdar://24029542 "Postfix '.' is reserved" error message" isn't helpful

This adds some heuristics so we can emit a fixit to remove extraneous
whitespace after a . and diagnose the case where a member just hasn't
been written yet better.  This also improves handling of tok::unknown
throughout the parser a bit.

include/swift/AST/DiagnosticsParse.def

@@ -132,8 +132,8 @@ ERROR(lex_unexpected_block_comment_end,lexing,none,
       "unexpected end of block comment", ())
 ERROR(lex_unary_equal,lexing,none,
       "'=' must have consistent whitespace on both sides", ())
-ERROR(lex_unary_postfix_dot_is_reserved,lexing,none,
-      "postfix '.' is reserved", ())
+ERROR(extra_whitespace_period,lexing,none,
+      "extraneous whitespace after '.' is not permitted", ())
 ERROR(lex_editor_placeholder,lexing,none,
       "editor placeholder in source file", ())
 WARNING(lex_editor_placeholder_in_playground,lexing,none,

lib/Parse/Lexer.cpp

@@ -648,14 +648,45 @@ void Lexer::lexOperatorIdentifier() {
       if (leftBound == rightBound || leftBound)
         break;
       return formToken(tok::amp_prefix, TokStart);
-    case '.':
+    case '.': {
       if (leftBound == rightBound)
         return formToken(tok::period, TokStart);
       if (rightBound)
         return formToken(tok::period_prefix, TokStart);
-      diagnose(TokStart, diag::lex_unary_postfix_dot_is_reserved);
-      // always emit 'tok::period' to avoid trickle down parse errors
-      return formToken(tok::period, TokStart);
+      
+      // If left bound but not right bound, handle some likely situations.
+      
+      // If there is just some horizontal whitespace before the next token, its
+      // addition is probably incorrect.
+      const char *AfterHorzWhitespace = CurPtr;
+      while (*AfterHorzWhitespace == ' ' || *AfterHorzWhitespace == '\t')
+        ++AfterHorzWhitespace;
+      
+      // First, when we are code completing "x.<ESC>", then make sure to return
+      // a tok::period, since that is what the user is wanting to know about.
+      // FIXME: isRightBound should consider this to be right bound.
+      if (*AfterHorzWhitespace == '\0' &&
+          AfterHorzWhitespace == CodeCompletionPtr) {
+        diagnose(TokStart, diag::expected_member_name);
+        return formToken(tok::period, TokStart);
+      }
+      
+      if (isRightBound(AfterHorzWhitespace, leftBound) &&
+          // Don't consider comments to be this.  A leading slash is probably
+          // either // or /* and most likely occurs just in our testsuite for
+          // expected-error lines.
+          *AfterHorzWhitespace != '/') {
+        diagnose(TokStart, diag::extra_whitespace_period)
+          .fixItRemoveChars(getSourceLoc(CurPtr),
+                            getSourceLoc(AfterHorzWhitespace));
+        return formToken(tok::period, TokStart);
+      }
+
+      // Otherwise, it is probably a missing member.
+      diagnose(TokStart, diag::expected_member_name);
+      //return formToken(tok::unknown, TokStart);
+      return lexImpl();
+    }
     case '?':
       if (leftBound)
         return formToken(tok::question_postfix, TokStart);

lib/Parse/ParseDecl.cpp

@@ -2004,6 +2004,10 @@ ParserStatus Parser::parseDecl(SmallVectorImpl<Decl*> &Entries,
       }
       diagnose(Tok, diag::expected_decl);
       return makeParserErrorResult<Decl>();
+  
+    case tok::unknown:
+      consumeToken(tok::unknown);
+      continue;
         
     // Unambiguous top level decls.
     case tok::kw_import:
@@ -4473,7 +4477,7 @@ bool Parser::parseNominalDeclMembers(SmallVectorImpl<Decl *> &memberDecls,
             /*AllowSepAfterLast=*/false, ErrorDiag, [&]() -> ParserStatus {
     // If the previous declaration didn't have a semicolon and this new
     // declaration doesn't start a line, complain.
-    if (!previousHadSemi && !Tok.isAtStartOfLine()) {
+    if (!previousHadSemi && !Tok.isAtStartOfLine() && !Tok.is(tok::unknown)) {
       SourceLoc endOfPrevious = getEndOfPreviousLoc();
       diagnose(endOfPrevious, diag::declaration_same_line_without_semi)
         .fixItInsert(endOfPrevious, ";");

lib/Parse/ParseExpr.cpp

@@ -610,6 +610,10 @@ ParserResult<Expr> Parser::parseExprSuper() {
     consumeToken(tok::code_complete);
     return makeParserCodeCompletionResult(superRef);
   }
+  
+  if (consumeIf(tok::unknown))
+    return nullptr;
+  
   diagnose(Tok, diag::expected_dot_or_subscript_after_super);
   return nullptr;
 }

lib/Parse/ParseStmt.cpp

@@ -258,6 +258,10 @@ ParserStatus Parser::parseBraceItems(SmallVectorImpl<ASTNode> &Entries,
         skipExtraTopLevelRBraces())
       continue;
 
+    // Eat invalid tokens instead of allowing them to produce downstream errors.
+    if (consumeIf(tok::unknown))
+      continue;
+           
     bool NeedParseErrorRecovery = false;
     ASTNode Result;
 

test/IDE/test-input-complete/test_input.swift

@@ -22,11 +22,11 @@
 // RUN: %swift-ide-test -test-input-complete -source-filename %S/Inputs/switch_incomplete3.swift | FileCheck %s -check-prefix=INCOMPLETE
 // RUN: %swift-ide-test -test-input-complete -source-filename %S/Inputs/toplevel_complete.swift | FileCheck %s -check-prefix=COMPLETE
 // RUN: %swift-ide-test -test-input-complete -source-filename %S/Inputs/toplevel_incomplete.swift | FileCheck %s -check-prefix=INCOMPLETE
-// RUN: %swift-ide-test -test-input-complete -source-filename %S/Inputs/toplevel_incomplete2.swift | FileCheck %s -check-prefix=INCOMPLETE
-// RUN: %swift-ide-test -test-input-complete -source-filename %S/Inputs/toplevel_incomplete3.swift | FileCheck %s -check-prefix=INCOMPLETE
+// RUN: %swift-ide-test -test-input-complete -source-filename %S/Inputs/toplevel_incomplete2.swift | FileCheck %s -check-prefix=COMPLETE
+// RUN: %swift-ide-test -test-input-complete -source-filename %S/Inputs/toplevel_incomplete3.swift | FileCheck %s -check-prefix=COMPLETE
 // RUN: %swift-ide-test -test-input-complete -source-filename %S/Inputs/toplevel_incomplete4.swift | FileCheck %s -check-prefix=INCOMPLETE
 // RUN: %swift-ide-test -test-input-complete -source-filename %S/Inputs/toplevel_incomplete5.swift | FileCheck %s -check-prefix=INCOMPLETE
-// RUN: %swift-ide-test -test-input-complete -source-filename %S/Inputs/type_incomplete1.swift | FileCheck %s -check-prefix=INCOMPLETE
+// RUN: %swift-ide-test -test-input-complete -source-filename %S/Inputs/type_incomplete1.swift | FileCheck %s -check-prefix=COMPLETE
 // RUN: %swift-ide-test -test-input-complete -source-filename %S/Inputs/type_incomplete2.swift | FileCheck %s -check-prefix=INCOMPLETE
 // RUN: %swift-ide-test -test-input-complete -source-filename %S/Inputs/type_incomplete3.swift | FileCheck %s -check-prefix=INCOMPLETE
 // RUN: %swift-ide-test -test-input-complete -source-filename %S/Inputs/type_incomplete4.swift | FileCheck %s -check-prefix=INCOMPLETE

test/Parse/recovery.swift

@@ -261,7 +261,7 @@ struct ErrorTypeInVarDecl1 {
 }
 
 struct ErrorTypeInVarDecl2 {
-  var v1 : Int. // expected-error {{expected identifier in dotted type}} expected-error {{postfix '.' is reserved}}
+  var v1 : Int. // expected-error {{expected member name following '.'}}
   var v2 : Int
 }
 
@@ -416,7 +416,7 @@ struct MissingInitializer1 {
 //===--- Recovery for expr-postfix.
 
 func exprPostfix1(x : Int) {
-  x. // expected-error {{postfix '.' is reserved}} expected-error {{expected member name following '.'}}
+  x. // expected-error {{expected member name following '.'}}
 }
 
 func exprPostfix2() {
@@ -435,7 +435,7 @@ class ExprSuper1 {
 
 class ExprSuper2 {
   init() {
-    super. // expected-error {{postfix '.' is reserved}} expected-error {{expected identifier or 'init' after super '.' expression}}
+    super. // expected-error {{expected member name following '.'}} expected-error {{expected '.' or '[' after 'super'}}
   }
 }
 
@@ -651,3 +651,11 @@ func r21712891(s : String) -> String {
   return "\(s[a)"  // expected-error 3 {{}}
 }
 
+
+// <rdar://problem/24029542> "Postfix '.' is reserved" error message" isn't helpful
+func postfixDot(a : String) {
+  _ = a.utf8
+  _ = a.   utf8  // expected-error {{extraneous whitespace after '.' is not permitted}} {{9-12=}}
+  _ = a.       // expected-error {{expected member name following '.'}}
+    a.         // expected-error {{expected member name following '.'}}
+}

test/Sema/diag_values_of_module_type.swift

@@ -95,13 +95,13 @@ func badTest2() {
   _ = x
 }
 func badTest3() {
-  var x = Swift. // expected-error {{postfix '.' is reserved}} expected-error {{expected member name following '.'}}
+  var _ = Swift. // expected-error {{expected member name following '.'}} expected-error {{expected module member name after module name}}
 }
 func badTest4() {
   Swift // expected-error {{expected module member name after module name}}
 }
 func badTest5() {
-  Swift. // expected-error {{postfix '.' is reserved}} expected-error {{expected member name following '.'}}
+  Swift. // expected-error {{expected module member name after module name}} expected-error {{expected member name following '.'}}
 }
 func badTest6() {
   _ = { () -> Int in

test/expr/expressions.swift

@@ -405,13 +405,13 @@ var st_u11 = " \u{00110000} "  // expected-error {{invalid unicode scalar}}
 func stringliterals(d: [String: Int]) {
 
   // rdar://11385385
-  var x = 4
+  let x = 4
   "Hello \(x+1) world"
   
   "Error: \(x+1"; // expected-error {{unterminated string literal}}
   
   "Error: \(x+1   // expected-error {{unterminated string literal}}
-  ;
+  ;    // expected-error {{';' statements are not allowed}}
 
   // rdar://14050788 [DF] String Interpolations can't contain quotes
   "test \("nested")"
@@ -432,6 +432,7 @@ func stringliterals(d: [String: Int]) {
   // expected-error @-2 {{unterminated string literal}} expected-error @-1 {{unterminated string literal}}
 
   // FIXME: bad diagnostics.
+  // expected-warning @+1 {{initialization of variable 'x2' was never used; consider replacing with assignment to '_' or removing it}}
   /* expected-error {{unterminated string literal}} expected-error {{expected ',' separator}} expected-error {{expected ',' separator}} expected-note {{to match this opening '('}}  */ var x2 : () = ("hello" + "
   ; // expected-error {{expected expression in list of expressions}}
 } // expected-error {{expected ')' in expression list}}