[libSyntax] Remove the C++ SyntaxClassifier

The recommended way forward is to use the SyntaxClassifier on the Swift
side.

By removing the C++ SyntaxClassifier, we can also eliminate the
-force-libsyntax-based-processing option that was used to bootstrap
incremental parsing and would generate the syntax map from a syntax
tree.

include/swift/Syntax/CMakeLists.txt

@@ -8,7 +8,6 @@ set(generated_include_sources
     SyntaxKind.h.gyb
     SyntaxNodes.h.gyb
     SyntaxBuilders.h.gyb
-    SyntaxClassifier.h.gyb
     SyntaxFactory.h.gyb
     SyntaxVisitor.h.gyb
     Trivia.h.gyb)

include/swift/Syntax/SyntaxClassifier.h.gyb

@@ -1,118 +0,0 @@
-%{
-  # -*- mode: C++ -*-
-  from gyb_syntax_support import *
-  NODE_MAP = create_node_map()
-  # Ignore the following admonition; it applies to the resulting .h file only
-}%
-//// Automatically Generated From SyntaxClassifier.h.gyb.
-//// Do Not Edit Directly!
-//===----------- SyntaxClassifier.h - SyntaxClassifier definitions --------===//
-//
-// This source file is part of the Swift.org open source project
-//
-// Copyright (c) 2014 - 2018 Apple Inc. and the Swift project authors
-// Licensed under Apache License v2.0 with Runtime Library Exception
-//
-// See https://swift.org/LICENSE.txt for license information
-// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef SWIFT_SYNTAX_CLASSIFIER_H
-#define SWIFT_SYNTAX_CLASSIFIER_H
-
-#include "swift/Syntax/SyntaxVisitor.h"
-#include <stack>
-
-namespace swift {
-namespace syntax {
-
-
-/// A classification that determines which color a token should be colored in
-/// for syntax coloring.
-enum class SyntaxClassification {
-  None,
-  Keyword,
-  Identifier,
-  DollarIdentifier,
-  IntegerLiteral,
-  FloatingLiteral,
-  StringLiteral,
-  /// Marks the parens for a string interpolation.
-  StringInterpolationAnchor,
-  TypeIdentifier,
-  /// #if/#else/#endif occurrence.
-  BuildConfigKeyword,
-  /// An identifier in a #if condition.
-  BuildConfigId,
-  /// #-keywords like #warning, #sourceLocation
-  PoundDirectiveKeyword,
-  /// Any occurrence of '@<attribute-name>' anywhere.
-  Attribute,
-  /// An editor placeholder string <#like this#>.
-  EditorPlaceholder,
-  ObjectLiteral
-};
-
-
-class SyntaxClassifier: public SyntaxVisitor {
-  struct ContextStackEntry {
-    /// The classification all identifiers shall inherit
-    SyntaxClassification Classification;
-    /// If set to \c true, all tokens will be forced to receive the above
-    /// classification, overriding their context-free classification
-    bool ForceClassification;
-
-    ContextStackEntry(SyntaxClassification Classification,
-                      bool ForceClassification)
-      : Classification(Classification),
-        ForceClassification(ForceClassification) {}
-  };
-
-  std::map<SyntaxNodeId, SyntaxClassification> ClassifiedTokens;
-  /// The top classification of this stack determines the color of identifiers
-  std::stack<ContextStackEntry, llvm::SmallVector<ContextStackEntry, 16>> ContextStack;
-
-  template<typename T>
-  void visit(T Node, SyntaxClassification Classification,
-             bool ForceClassification) {
-    ContextStack.emplace(Classification, ForceClassification);
-    visit(Node);
-    ContextStack.pop();
-  }
-
-  template<typename T>
-  void visit(llvm::Optional<T> OptNode) {
-    if (OptNode.hasValue()) {
-      static_cast<SyntaxVisitor *>(this)->visit(OptNode.getValue());
-    }
-  }
-
-  virtual void visit(TokenSyntax TokenNode) override;
-
-  virtual void visit(Syntax Node) override {
-    SyntaxVisitor::visit(Node);
-  }
-
-% for node in SYNTAX_NODES:
-%   if is_visitable(node):
-  virtual void visit(${node.name} Node) override;
-%   end
-% end
-
-public:
-  std::map<SyntaxNodeId, SyntaxClassification> classify(Syntax Node) {
-    // Clean up the environment
-    ContextStack = std::stack<ContextStackEntry, llvm::SmallVector<ContextStackEntry, 16>>();
-    ContextStack.push({SyntaxClassification::None, false});
-    ClassifiedTokens.clear();
-
-    Node.accept(*this);
-
-    return ClassifiedTokens;
-  }
-};
-} // namespace syntax
-} // namespace swift
-
-#endif // SWIFT_SYNTAX_CLASSIFIER_H

lib/Syntax/CMakeLists.txt

@@ -14,6 +14,5 @@ add_swift_library(swiftSyntax STATIC
   RawSyntax.cpp
   Syntax.cpp
   SyntaxArena.cpp
-  SyntaxClassifier.cpp.gyb
   SyntaxData.cpp
   UnknownSyntax.cpp)

lib/Syntax/SyntaxClassifier.cpp.gyb

@@ -1,160 +0,0 @@
-%{
-  from gyb_syntax_support import *
-  # -*- mode: C++ -*-
-  # Ignore the following admonition; it applies to the resulting .cpp file only
-}%
-//// Automatically Generated From SyntaxClassifier.cpp.gyb.
-//// Do Not Edit Directly!
-//===----- SyntaxClassifier.cpp - Syntax Classifier implementations -------===//
-//
-// This source file is part of the Swift.org open source project
-//
-// Copyright (c) 2014 - 2018 Apple Inc. and the Swift project authors
-// Licensed under Apache License v2.0 with Runtime Library Exception
-//
-// See https://swift.org/LICENSE.txt for license information
-// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines the Syntax Classifier, which walks the syntax tree and
-// creates a classification table for all tokens in the syntax tree, mapping it
-// to a \c SyntaxClassification by its ID.
-//
-//===----------------------------------------------------------------------===//
-
-#include "swift/Basic/EditorPlaceholder.h"
-#include "swift/Syntax/SyntaxClassifier.h"
-
-using namespace swift;
-using namespace swift::syntax;
-
-% for node in SYNTAX_NODES:
-%   if is_visitable(node):
-void SyntaxClassifier::visit(${node.name} Node) {
-%     if node.is_unknown() or node.is_syntax_collection():
-  SyntaxVisitor::visit(Node);
-%     else:
-%       for child in node.children:
-%         if child.classification:
-  visit(Node.get${child.name}(), SyntaxClassification::${child.classification}, ${"true" if child.force_classification else "false"});
-%         else:
-  visit(Node.get${child.name}());
-%         end
-%       end
-%     end
-}
-%   end
-% end
-
-/// Returns the SyntaxClassficiation a token node should receive if it is not
-/// inside a special context. Returns \c None if the token has no context-free
-/// classification and should always inherit from the context.
-llvm::Optional<SyntaxClassification>
-getContextFreeClassificationForToken(TokenSyntax TokenNode) {
-  switch (TokenNode.getTokenKind()) {
-#define KEYWORD(KW) case tok::kw_##KW: return SyntaxClassification::Keyword;
-#define POUND_KEYWORD(KW) case tok::pound_##KW: return SyntaxClassification::Keyword;
-#define POUND_OBJECT_LITERAL(KW, desc, proto) case tok::pound_##KW: return SyntaxClassification::ObjectLiteral;
-#define POUND_DIRECTIVE_KEYWORD(KW) case tok::pound_##KW: return SyntaxClassification::PoundDirectiveKeyword;
-#define POUND_COND_DIRECTIVE_KEYWORD(KW) case tok::pound_##KW: return SyntaxClassification::BuildConfigKeyword;
-#include "swift/Syntax/TokenKinds.def"
-  // Punctuators
-  case tok::l_paren:
-  case tok::r_paren:
-  case tok::l_brace:
-  case tok::r_brace:
-  case tok::l_square:
-  case tok::r_square:
-  case tok::l_angle:
-  case tok::r_angle:
-  case tok::period:
-  case tok::period_prefix:
-  case tok::comma:
-  case tok::colon:
-  case tok::semi:
-  case tok::equal:
-  case tok::pound:
-  case tok::amp_prefix:
-  case tok::arrow:
-  case tok::backtick:
-  case tok::backslash:
-  case tok::exclaim_postfix:
-  case tok::question_postfix:
-  case tok::question_infix:
-  case tok::sil_dollar:
-  case tok::sil_exclamation:
-    return SyntaxClassification::None;
-  case tok::string_quote:
-  case tok::multiline_string_quote:
-    return SyntaxClassification::StringLiteral;
-  case tok::at_sign:
-    return SyntaxClassification::Attribute;
-
-  // Literals
-  case tok::integer_literal:
-    return SyntaxClassification::IntegerLiteral;
-  case tok::floating_literal:
-    return SyntaxClassification::FloatingLiteral;
-  case tok::string_literal:
-    return SyntaxClassification::StringLiteral;
-
-  // Miscelaneous
-  case tok::identifier: {
-    if (isEditorPlaceholder(TokenNode.getText())) {
-      return SyntaxClassification::EditorPlaceholder;
-    } else {
-      return llvm::None;
-    }
-  }
-  case tok::unknown:
-    if (TokenNode.getText().startswith("\"")) {
-      // Unterminated string literal
-      return SyntaxClassification::StringLiteral;
-    } else {
-      return SyntaxClassification::None;
-    }
-    break;
-  case tok::eof:
-  case tok::code_complete:
-  case tok::oper_binary_unspaced:
-  case tok::oper_binary_spaced:
-  case tok::oper_postfix:
-  case tok::oper_prefix:
-    return SyntaxClassification::None;
-  case tok::dollarident:
-    return SyntaxClassification::DollarIdentifier;
-  case tok::sil_local_name:
-    return SyntaxClassification::None;
-  case tok::comment:
-    llvm_unreachable("Comments should be in trivia");
-  case tok::contextual_keyword:
-  case tok::kw_yield:
-    return SyntaxClassification::Keyword;
-  case tok::string_segment:
-    return SyntaxClassification::StringLiteral;
-  case tok::string_interpolation_anchor:
-    return SyntaxClassification::StringInterpolationAnchor;
-  case tok::NUM_TOKENS:
-    llvm_unreachable("");
-  }
-}
-
-void SyntaxClassifier::visit(TokenSyntax TokenNode) {
-  SyntaxClassification Classification = ContextStack.top().Classification;
-  bool ForceClassification = ContextStack.top().ForceClassification;
-  if (!ForceClassification) {
-    auto NativeClassification = getContextFreeClassificationForToken(TokenNode);
-    if (NativeClassification.hasValue()) {
-      Classification = NativeClassification.getValue();
-    }
-    if (Classification == SyntaxClassification::None &&
-        TokenNode.getTokenKind() == tok::identifier) {
-      Classification = SyntaxClassification::Identifier;
-    }
-  }
-
-  assert(ClassifiedTokens.count(TokenNode.getId()) == 0 &&
-         "Token already classified");
-  ClassifiedTokens[TokenNode.getId()] = Classification;
-}

test/IDE/coloring.swift

@@ -1,6 +1,5 @@
 // RUN: %target-swift-ide-test -syntax-coloring -source-filename %s | %FileCheck %s
-// RUN: %target-swift-ide-test -syntax-coloring -typecheck -source-filename %s | %FileCheck %s -check-prefix CHECK -check-prefix CHECK-OLD
-// RUN: %target-swift-ide-test -syntax-coloring -force-libsyntax-based-processing -source-filename %s | %FileCheck %s --check-prefix CHECK -check-prefix CHECK-NEW
+// RUN: %target-swift-ide-test -syntax-coloring -typecheck -source-filename %s | %FileCheck %s -check-prefixes CHECK,CHECK-OLD
 // XFAIL: broken_std_regex
 
 #line 17 "abc.swift"

test/IDE/coloring_configs.swift

@@ -1,5 +1,4 @@
 // RUN: %target-swift-ide-test -syntax-coloring -source-filename %s -D CONF | %FileCheck %s
-// RUN: %target-swift-ide-test -syntax-coloring -force-libsyntax-based-processing -source-filename %s -D CONF | %FileCheck %s
 
 // CHECK: <kw>var</kw> f : <type>Int</type>
 var f : Int

test/IDE/coloring_keywords.swift

@@ -1,5 +1,4 @@
 // RUN: %target-swift-ide-test -syntax-coloring -source-filename %s | %FileCheck %s
-// RUN: %target-swift-ide-test -syntax-coloring -force-libsyntax-based-processing -source-filename %s | %FileCheck %s
 // RUN: %target-swift-ide-test -syntax-coloring -typecheck -source-filename %s | %FileCheck %s
 
 // CHECK: <kw>return</kw> c.return

test/IDE/coloring_unclosed_hash_if.swift

@@ -1,5 +1,4 @@
 // RUN: %target-swift-ide-test -syntax-coloring -source-filename %s | %FileCheck %s
-// RUN: %target-swift-ide-test -syntax-coloring -force-libsyntax-based-processing -source-filename %s | %FileCheck %s
 // RUN: %target-swift-ide-test -syntax-coloring -typecheck -source-filename %s | %FileCheck %s
 
 // CHECK: <#kw>#if</#kw> <#id>d</#id>

test/SourceKit/SyntaxMapData/syntaxmap-edit-block-comment.swift

@@ -1,7 +1,5 @@
 // RUN: %sourcekitd-test -req=open -print-raw-response %S/Inputs/syntaxmap-edit-block-comment.swift == -req=edit -print-raw-response %S/Inputs/syntaxmap-edit-block-comment.swift -pos=4:2 -replace=" " -length=1  == -req=edit -print-raw-response %S/Inputs/syntaxmap-edit-block-comment.swift -pos=4:2 -replace="/" -length=1 == -req=edit -print-raw-response %S/Inputs/syntaxmap-edit-block-comment.swift -pos=1:1 -replace="//" -length=2 | %sed_clean > %t.response
-// RUN: %FileCheck -input-file=%t.response %s --check-prefixes CHECK,CHECK-OLD
-// RUN: %sourcekitd-test -req=open -print-raw-response %S/Inputs/syntaxmap-edit-block-comment.swift -force-libsyntax-based-processing == -req=edit -print-raw-response %S/Inputs/syntaxmap-edit-block-comment.swift -pos=4:2 -replace=" " -length=1 -force-libsyntax-based-processing  == -req=edit -print-raw-response %S/Inputs/syntaxmap-edit-block-comment.swift -pos=4:2 -replace="/" -length=1 -force-libsyntax-based-processing == -req=edit -print-raw-response %S/Inputs/syntaxmap-edit-block-comment.swift -pos=1:1 -replace="//" -length=2 -force-libsyntax-based-processing | %sed_clean > %t.libSyntax.response
-// RUN: %FileCheck -input-file=%t.libSyntax.response %s --check-prefixes CHECK,CHECK-NEW
+// RUN: %FileCheck -input-file=%t.response %s
 
 // Initial state
 
@@ -89,14 +87,12 @@
 
 // CHECK: {{^}}{
 // CHECK-NEXT: key.offset: 0,
-// CHECK-OLD-NEXT: key.length: 3,
-// CHECK-NEW-NEXT: key.length: 2,
+// CHECK-NEXT: key.length: 3,
 // CHECK-NEXT: key.diagnostic_stage: source.diagnostic.stage.swift.parse,
 // CHECK-NEXT: key.syntaxmap: [
 // CHECK-NEXT:   {
 // CHECK-NEXT:     key.kind: source.lang.swift.syntaxtype.comment,
 // CHECK-NEXT:     key.offset: 0,
-// CHECK-OLD-NEXT:     key.length: 3
-// CHECK-NEW-NEXT:     key.length: 2
+// CHECK-NEXT:     key.length: 3
 // CHECK-NEXT:   }
 // CHECK-NEXT: ],

test/SourceKit/SyntaxMapData/syntaxmap-edit-chained-comment.swift

@@ -1,7 +1,5 @@
 // RUN: %sourcekitd-test -req=open -print-raw-response %S/Inputs/syntaxmap-edit-chained-comment.swift == -req=edit -print-raw-response %S/Inputs/syntaxmap-edit-chained-comment.swift -pos=1:9 -replace=" " -length=1  == -req=edit -print-raw-response %S/Inputs/syntaxmap-edit-chained-comment.swift -pos=1:9 -replace="/" -length=1 | %sed_clean > %t.response
-// RUN: %FileCheck -input-file=%t.response %s --check-prefixes CHECK,CHECK-OLD
-// RUN: %sourcekitd-test -req=open -print-raw-response %S/Inputs/syntaxmap-edit-chained-comment.swift -force-libsyntax-based-processing == -req=edit -print-raw-response %S/Inputs/syntaxmap-edit-chained-comment.swift -pos=1:9 -replace=" " -length=1 -force-libsyntax-based-processing  == -req=edit -print-raw-response %S/Inputs/syntaxmap-edit-chained-comment.swift -pos=1:9 -replace="/" -length=1 -force-libsyntax-based-processing | %sed_clean > %t.libsyntax.response
-// RUN: %FileCheck -input-file=%t.libsyntax.response %s --check-prefixes CHECK,CHECK-NEW
+// RUN: %FileCheck -input-file=%t.response %s
 
 // Initial state
 
@@ -48,8 +46,7 @@
 // CHECK-NEXT:   {
 // CHECK-NEXT:     key.kind: source.lang.swift.syntaxtype.comment,
 // CHECK-NEXT:     key.offset: 13,
-// CHECK-OLD-NEXT:     key.length: 6
-// CHECK-NEW-NEXT:     key.length: 5
+// CHECK-NEXT:     key.length: 6
 // CHECK-NEXT:   }
 // CHECK-NEXT: ],
 

test/SourceKit/SyntaxMapData/syntaxmap-edit-del.swift

@@ -2,8 +2,3 @@
 // RUN: diff -u %s.response %t.response
 // RUN: %sourcekitd-test -req=syntax-map -pos=4:1 -length=2 -replace="" %S/Inputs/syntaxmap-edit-del.swift | %sed_clean > %t.response2
 // RUN: diff -u %s.response2 %t.response2
-
-// RUN: %sourcekitd-test -req=syntax-map -pos=2:1 -length=2 -replace=" " %S/Inputs/syntaxmap-edit-del.swift -force-libsyntax-based-processing | %sed_clean > %t.libsyntax.response
-// RUN: diff -u %s.response %t.libsyntax.response
-// RUN: %sourcekitd-test -req=syntax-map -pos=4:1 -length=2 -replace="" %S/Inputs/syntaxmap-edit-del.swift -force-libsyntax-based-processing | %sed_clean > %t.libsyntax.response2
-// RUN: diff -u %s.response2 %t.libsyntax.response2

test/SourceKit/SyntaxMapData/syntaxmap-edit-disjoint-effect.swift

@@ -1,7 +1,5 @@
 // RUN: %sourcekitd-test -req=open -print-raw-response %S/Inputs/syntaxmap-edit-disjoint-effect.swift == -req=edit -print-raw-response %S/Inputs/syntaxmap-edit-disjoint-effect.swift -pos=1:11 -replace='(' -length=1 | %sed_clean > %t.response
 // RUN: %FileCheck -input-file=%t.response %s
-// RUN: %sourcekitd-test -req=open -print-raw-response %S/Inputs/syntaxmap-edit-disjoint-effect.swift -force-libsyntax-based-processing == -req=edit -print-raw-response %S/Inputs/syntaxmap-edit-disjoint-effect.swift -pos=1:11 -replace='(' -length=1 -force-libsyntax-based-processing | %sed_clean > %t.libsyntax.response
-// RUN: %FileCheck -input-file=%t.libsyntax.response %s
 
 // Original contents
 

test/SourceKit/SyntaxMapData/syntaxmap-edit-multiline-string.swift

@@ -1,7 +1,5 @@
 // RUN: %sourcekitd-test -req=open -print-raw-response %S/Inputs/syntaxmap-edit-multiline-string.swift == -req=edit -print-raw-response %S/Inputs/syntaxmap-edit-multiline-string.swift -pos=8:1 -replace='"""' -length=3 == -req=edit -print-raw-response %S/Inputs/syntaxmap-edit-multiline-string.swift -pos=6:2 -replace=')' -length=1 == -req=edit -print-raw-response %S/Inputs/syntaxmap-edit-multiline-string.swift -pos=2:10 -replace=' ' -length=1 | %sed_clean > %t.response
-// RUN: %FileCheck -input-file=%t.response %s --check-prefixes CHECK,CHECK-OLD
-// RUN: %sourcekitd-test -req=open -print-raw-response %S/Inputs/syntaxmap-edit-multiline-string.swift -force-libsyntax-based-processing == -req=edit -print-raw-response %S/Inputs/syntaxmap-edit-multiline-string.swift -pos=8:1 -replace='"""' -length=3 -force-libsyntax-based-processing == -req=edit -print-raw-response %S/Inputs/syntaxmap-edit-multiline-string.swift -pos=6:2 -replace=')' -length=1 -force-libsyntax-based-processing == -req=edit -print-raw-response %S/Inputs/syntaxmap-edit-multiline-string.swift -pos=2:10 -replace=' ' -length=1 -force-libsyntax-based-processing | %sed_clean > %t.libsyntax.response
-// RUN: %FileCheck -input-file=%t.libsyntax.response %s --check-prefixes CHECK,CHECK-NEW
+// RUN: %FileCheck -input-file=%t.response %s
 
 // Original file contents
 
@@ -52,14 +50,8 @@
 // CHECK-NEXT:   {
 // CHECK-NEXT:     key.kind: source.lang.swift.syntaxtype.string,
 // CHECK-NEXT:     key.offset: 24,
-// CHECK-OLD-NEXT:     key.length: 5
-// CHECK-NEW-NEXT:     key.length: 3
+// CHECK-NEXT:     key.length: 5
 // CHECK-NEXT:   },
-// CHECK-NEW-NEXT:   {
-// CHECK-NEW-NEXT:     key.kind: source.lang.swift.syntaxtype.string,
-// CHECK-NEW-NEXT:     key.offset: 27,
-// CHECK-NEW-NEXT:     key.length: 2
-// CHECK-NEW-NEXT:   },
 // CHECK-NEXT:   {
 // CHECK-NEXT:     key.kind: source.lang.swift.syntaxtype.string_interpolation_anchor,
 // CHECK-NEXT:     key.offset: 30,
@@ -78,14 +70,8 @@
 // CHECK-NEXT:   {
 // CHECK-NEXT:     key.kind: source.lang.swift.syntaxtype.string,
 // CHECK-NEXT:     key.offset: 35,
-// CHECK-OLD-NEXT:     key.length: 6
-// CHECK-NEW-NEXT:     key.length: 3
+// CHECK-NEXT:     key.length: 6
 // CHECK-NEXT:   },
-// CHECK-NEW-NEXT:   {
-// CHECK-NEW-NEXT:     key.kind: source.lang.swift.syntaxtype.string,
-// CHECK-NEW-NEXT:     key.offset: 38,
-// CHECK-NEW-NEXT:     key.length: 3
-// CHECK-NEW-NEXT:   },
 // CHECK-NEXT:   {
 // CHECK-NEXT:     key.kind: source.lang.swift.syntaxtype.keyword,
 // CHECK-NEXT:     key.offset: 43,
@@ -116,15 +102,13 @@
 // After adding a character after the interpolation
 // CHECK: {{^}}{
 // CHECK-NEXT: key.offset: 35,
-// CHECK-OLD-NEXT: key.length: 6,
-// CHECK-NEW-NEXT: key.length: 3,
+// CHECK-NEXT: key.length: 6,
 // CHECK-NEXT: key.diagnostic_stage: source.diagnostic.stage.swift.parse,
 // CHECK-NEXT: key.syntaxmap: [
 // CHECK-NEXT:   {
 // CHECK-NEXT:     key.kind: source.lang.swift.syntaxtype.string,
 // CHECK-NEXT:     key.offset: 35,
-// CHECK-OLD-NEXT:     key.length: 6
-// CHECK-NEW-NEXT:     key.length: 3
+// CHECK-NEXT:     key.length: 6
 // CHECK-NEXT:   }
 // CHECK-NEXT: ],
 

test/SourceKit/SyntaxMapData/syntaxmap-edit-nested-token.swift

@@ -1,7 +1,5 @@
 // RUN: %sourcekitd-test -req=open -print-raw-response %S/Inputs/syntaxmap-edit-nested-token.swift == -req=edit -print-raw-response %S/Inputs/syntaxmap-edit-nested-token.swift -pos=10:43 -replace='impact' -length=6 | %sed_clean > %t.response
-// RUN: %FileCheck -input-file=%t.response %s --check-prefixes CHECK,CHECK-OLD
-// RUN: %sourcekitd-test -req=open -print-raw-response %S/Inputs/syntaxmap-edit-nested-token.swift -force-libsyntax-based-processing == -req=edit -print-raw-response %S/Inputs/syntaxmap-edit-nested-token.swift -pos=10:43 -replace='impact' -length=6 -force-libsyntax-based-processing | %sed_clean > %t.libsyntax.response
-// RUN: %FileCheck -input-file=%t.libsyntax.response %s --check-prefixes CHECK,CHECK-NEW
+// RUN: %FileCheck -input-file=%t.response %s
 
 // Original file contents
 
@@ -13,61 +11,48 @@
 // CHECK-NEXT:   {
 // CHECK-NEXT:     key.kind: source.lang.swift.syntaxtype.doccomment,
 // CHECK-NEXT:     key.offset: 0,
-// CHECK-OLD-NEXT:     key.length: 29
-// CHECK-NEW-NEXT:     key.length: 28
+// CHECK-NEXT:     key.length: 29
 // CHECK-NEXT:   },
 // CHECK-NEXT:   {
 // CHECK-NEXT:     key.kind: source.lang.swift.syntaxtype.doccomment,
 // CHECK-NEXT:     key.offset: 29,
-// CHECK-OLD-NEXT:     key.length: 4
-// CHECK-NEW-NEXT:     key.length: 3
+// CHECK-NEXT:     key.length: 4
+// CHECK-NEXT:   },
+// CHECK-NEXT:   {
+// CHECK-NEXT:     key.kind: source.lang.swift.syntaxtype.doccomment,
+// CHECK-NEXT:     key.offset: 33,
+// CHECK-NEXT:     key.length: 6
+// CHECK-NEXT:   },
+// CHECK-NEXT:   {
+// CHECK-NEXT:     key.kind: source.lang.swift.syntaxtype.doccomment.field,
+// CHECK-NEXT:     key.offset: 39,
+// CHECK-NEXT:     key.length: 9
+// CHECK-NEXT:   },
+// CHECK-NEXT:   {
+// CHECK-NEXT:     key.kind: source.lang.swift.syntaxtype.doccomment,
+// CHECK-NEXT:     key.offset: 48,
+// CHECK-NEXT:     key.length: 28
 // CHECK-NEXT:   },
-// CHECK-OLD-NEXT:   {
-// CHECK-OLD-NEXT:     key.kind: source.lang.swift.syntaxtype.doccomment,
-// CHECK-OLD-NEXT:     key.offset: 33,
-// CHECK-OLD-NEXT:     key.length: 6
-// CHECK-OLD-NEXT:   },
-// CHECK-OLD-NEXT:   {
-// CHECK-OLD-NEXT:     key.kind: source.lang.swift.syntaxtype.doccomment.field,
-// CHECK-OLD-NEXT:     key.offset: 39,
-// CHECK-OLD-NEXT:     key.length: 9
-// CHECK-OLD-NEXT:   },
-// CHECK-OLD-NEXT:   {
-// CHECK-OLD-NEXT:     key.kind: source.lang.swift.syntaxtype.doccomment,
-// CHECK-OLD-NEXT:     key.offset: 48,
-// CHECK-OLD-NEXT:     key.length: 28
-// CHECK-OLD-NEXT:   },
-// CHECK-NEW-NEXT:   {
-// CHECK-NEW-NEXT:     key.kind: source.lang.swift.syntaxtype.doccomment,
-// CHECK-NEW-NEXT:     key.offset: 33,
-// CHECK-NEW-NEXT:     key.length: 42
-// CHECK-NEW-NEXT:   },
 // CHECK-NEXT:   {
 // CHECK-NEXT:     key.kind: source.lang.swift.syntaxtype.doccomment,
 // CHECK-NEXT:     key.offset: 76,
-// CHECK-OLD-NEXT:     key.length: 4
-// CHECK-NEW-NEXT:     key.length: 3
+// CHECK-NEXT:     key.length: 4
+// CHECK-NEXT:   },
+// CHECK-NEXT:   {
+// CHECK-NEXT:     key.kind: source.lang.swift.syntaxtype.doccomment,
+// CHECK-NEXT:     key.offset: 80,
+// CHECK-NEXT:     key.length: 6
+// CHECK-NEXT:   },
+// CHECK-NEXT:   {
+// CHECK-NEXT:     key.kind: source.lang.swift.syntaxtype.doccomment.field,
+// CHECK-NEXT:     key.offset: 86,
+// CHECK-NEXT:     key.length: 7
+// CHECK-NEXT:   },
+// CHECK-NEXT:   {
+// CHECK-NEXT:     key.kind: source.lang.swift.syntaxtype.doccomment,
+// CHECK-NEXT:     key.offset: 93,
+// CHECK-NEXT:     key.length: 19
 // CHECK-NEXT:   },
-// CHECK-OLD-NEXT:   {
-// CHECK-OLD-NEXT:     key.kind: source.lang.swift.syntaxtype.doccomment,
-// CHECK-OLD-NEXT:     key.offset: 80,
-// CHECK-OLD-NEXT:     key.length: 6
-// CHECK-OLD-NEXT:   },
-// CHECK-OLD-NEXT:   {
-// CHECK-OLD-NEXT:     key.kind: source.lang.swift.syntaxtype.doccomment.field,
-// CHECK-OLD-NEXT:     key.offset: 86,
-// CHECK-OLD-NEXT:     key.length: 7
-// CHECK-OLD-NEXT:   },
-// CHECK-OLD-NEXT:   {
-// CHECK-OLD-NEXT:     key.kind: source.lang.swift.syntaxtype.doccomment,
-// CHECK-OLD-NEXT:     key.offset: 93,
-// CHECK-OLD-NEXT:     key.length: 19
-// CHECK-OLD-NEXT:   },
-// CHECK-NEW-NEXT:   {
-// CHECK-NEW-NEXT:     key.kind: source.lang.swift.syntaxtype.doccomment,
-// CHECK-NEW-NEXT:     key.offset: 80,
-// CHECK-NEW-NEXT:     key.length: 31
-// CHECK-NEW-NEXT:   },
 // CHECK-NEXT:   {
 // CHECK-NEXT:     key.kind: source.lang.swift.syntaxtype.keyword,
 // CHECK-NEXT:     key.offset: 112,
@@ -121,61 +106,48 @@
 // CHECK-NEXT:   {
 // CHECK-NEXT:     key.kind: source.lang.swift.syntaxtype.doccomment,
 // CHECK-NEXT:     key.offset: 221,
-// CHECK-OLD-NEXT:     key.length: 35
-// CHECK-NEW-NEXT:     key.length: 34
+// CHECK-NEXT:     key.length: 35
 // CHECK-NEXT:   },
 // CHECK-NEXT:   {
 // CHECK-NEXT:     key.kind: source.lang.swift.syntaxtype.doccomment,
 // CHECK-NEXT:     key.offset: 256,
-// CHECK-OLD-NEXT:     key.length: 4
-// CHECK-NEW-NEXT:     key.length: 3
+// CHECK-NEXT:     key.length: 4
+// CHECK-NEXT:   },
+// CHECK-NEXT:   {
+// CHECK-NEXT:     key.kind: source.lang.swift.syntaxtype.doccomment,
+// CHECK-NEXT:     key.offset: 260,
+// CHECK-NEXT:     key.length: 6
+// CHECK-NEXT:   },
+// CHECK-NEXT:   {
+// CHECK-NEXT:     key.kind: source.lang.swift.syntaxtype.doccomment.field,
+// CHECK-NEXT:     key.offset: 266,
+// CHECK-NEXT:     key.length: 9
+// CHECK-NEXT:   },
+// CHECK-NEXT:   {
+// CHECK-NEXT:     key.kind: source.lang.swift.syntaxtype.doccomment,
+// CHECK-NEXT:     key.offset: 275,
+// CHECK-NEXT:     key.length: 28
 // CHECK-NEXT:   },
-// CHECK-OLD-NEXT:   {
-// CHECK-OLD-NEXT:     key.kind: source.lang.swift.syntaxtype.doccomment,
-// CHECK-OLD-NEXT:     key.offset: 260,
-// CHECK-OLD-NEXT:     key.length: 6
-// CHECK-OLD-NEXT:   },
-// CHECK-OLD-NEXT:   {
-// CHECK-OLD-NEXT:     key.kind: source.lang.swift.syntaxtype.doccomment.field,
-// CHECK-OLD-NEXT:     key.offset: 266,
-// CHECK-OLD-NEXT:     key.length: 9
-// CHECK-OLD-NEXT:   },
-// CHECK-OLD-NEXT:   {
-// CHECK-OLD-NEXT:     key.kind: source.lang.swift.syntaxtype.doccomment,
-// CHECK-OLD-NEXT:     key.offset: 275,
-// CHECK-OLD-NEXT:     key.length: 28
-// CHECK-OLD-NEXT:   },
-// CHECK-NEW-NEXT:   {
-// CHECK-NEW-NEXT:     key.kind: source.lang.swift.syntaxtype.doccomment,
-// CHECK-NEW-NEXT:     key.offset: 260,
-// CHECK-NEW-NEXT:     key.length: 42
-// CHECK-NEW-NEXT:   },
 // CHECK-NEXT:   {
 // CHECK-NEXT:     key.kind: source.lang.swift.syntaxtype.doccomment,
 // CHECK-NEXT:     key.offset: 303,
-// CHECK-OLD-NEXT:     key.length: 4
-// CHECK-NEW-NEXT:     key.length: 3
+// CHECK-NEXT:     key.length: 4
+// CHECK-NEXT:   },
+// CHECK-NEXT:   {
+// CHECK-NEXT:     key.kind: source.lang.swift.syntaxtype.doccomment,
+// CHECK-NEXT:     key.offset: 307,
+// CHECK-NEXT:     key.length: 6
+// CHECK-NEXT:   },
+// CHECK-NEXT:   {
+// CHECK-NEXT:     key.kind: source.lang.swift.syntaxtype.doccomment.field,
+// CHECK-NEXT:     key.offset: 313,
+// CHECK-NEXT:     key.length: 7
+// CHECK-NEXT:   },
+// CHECK-NEXT:   {
+// CHECK-NEXT:     key.kind: source.lang.swift.syntaxtype.doccomment,
+// CHECK-NEXT:     key.offset: 320,
+// CHECK-NEXT:     key.length: 19
 // CHECK-NEXT:   },
-// CHECK-OLD-NEXT:   {
-// CHECK-OLD-NEXT:     key.kind: source.lang.swift.syntaxtype.doccomment,
-// CHECK-OLD-NEXT:     key.offset: 307,
-// CHECK-OLD-NEXT:     key.length: 6
-// CHECK-OLD-NEXT:   },
-// CHECK-OLD-NEXT:   {
-// CHECK-OLD-NEXT:     key.kind: source.lang.swift.syntaxtype.doccomment.field,
-// CHECK-OLD-NEXT:     key.offset: 313,
-// CHECK-OLD-NEXT:     key.length: 7
-// CHECK-OLD-NEXT:   },
-// CHECK-OLD-NEXT:   {
-// CHECK-OLD-NEXT:     key.kind: source.lang.swift.syntaxtype.doccomment,
-// CHECK-OLD-NEXT:     key.offset: 320,
-// CHECK-OLD-NEXT:     key.length: 19
-// CHECK-OLD-NEXT:   },
-// CHECK-NEW-NEXT:   {
-// CHECK-NEW-NEXT:     key.kind: source.lang.swift.syntaxtype.doccomment,
-// CHECK-NEW-NEXT:     key.offset: 307,
-// CHECK-NEW-NEXT:     key.length: 31
-// CHECK-NEW-NEXT:   },
 // CHECK-NEXT:   {
 // CHECK-NEXT:     key.kind: source.lang.swift.syntaxtype.keyword,
 // CHECK-NEXT:     key.offset: 339,

test/SourceKit/SyntaxMapData/syntaxmap-edit-remove.swift

@@ -1,7 +1,5 @@
 // RUN: %sourcekitd-test -req=open -print-raw-response %S/Inputs/syntaxmap-edit-remove.swift == -req=edit -print-raw-response %S/Inputs/syntaxmap-edit-remove.swift -pos=3:3 -length=1 -replace='' == -req=edit -print-raw-response %S/Inputs/syntaxmap-edit-remove.swift -pos=2:1 -replace='' -length=1 == -req=edit -print-raw-response %S/Inputs/syntaxmap-edit-remove.swift -pos=1:9 -length=1 -replace='' | %sed_clean > %t.response
 // RUN: %FileCheck -input-file=%t.response %s
-// RUN: %sourcekitd-test -req=open -print-raw-response %S/Inputs/syntaxmap-edit-remove.swift -force-libsyntax-based-processing == -req=edit -print-raw-response %S/Inputs/syntaxmap-edit-remove.swift -pos=3:3 -length=1 -replace='' -force-libsyntax-based-processing == -req=edit -print-raw-response %S/Inputs/syntaxmap-edit-remove.swift -pos=2:1 -replace='' -length=1 -force-libsyntax-based-processing == -req=edit -print-raw-response %S/Inputs/syntaxmap-edit-remove.swift -pos=1:9 -length=1 -replace='' -force-libsyntax-based-processing | %sed_clean > %t.libsyntax.response
-// RUN: %FileCheck -input-file=%t.libsyntax.response %s
 
 // Initial state
 

test/SourceKit/SyntaxMapData/syntaxmap-edit.swift

@@ -1,4 +1,2 @@
 // RUN: %sourcekitd-test -req=syntax-map -pos=4:10 -replace="Bar" %S/Inputs/syntaxmap-edit.swift > %t.response
 // RUN: diff -u %s.response %t.response
-// RUN: %sourcekitd-test -req=syntax-map -pos=4:10 -replace="Bar" %S/Inputs/syntaxmap-edit.swift -force-libsyntax-based-processing > %t.libsyntax.response
-// RUN: diff -u %s.libsyntax.response %t.libsyntax.response

test/SourceKit/SyntaxMapData/syntaxmap-multiple-edits.swift

@@ -1,7 +1,5 @@
 // RUN: %sourcekitd-test -req=open -print-raw-response %S/Inputs/syntaxmap-multiple-edits.swift == -req=edit -print-raw-response -pos=6:13 -length=1 -replace=" " %S/Inputs/syntaxmap-multiple-edits.swift == -req="edit" -pos=14:1 -length=0 -replace="let y = 2" -print-raw-response %S/Inputs/syntaxmap-multiple-edits.swift == -req="edit" -pos=8:10 -length=7 -replace='Int64 = 3; let z = 2' -print-raw-response %S/Inputs/syntaxmap-multiple-edits.swift == -req="edit" -pos=4:9 -length=2 -replace='50 * 95 - 100' -print-raw-response %S/Inputs/syntaxmap-multiple-edits.swift == -req="edit" -pos=1:1 -length=0 -replace='func firstFunc(x: Int) {}' -print-raw-response %S/Inputs/syntaxmap-multiple-edits.swift | %sed_clean > %t.response
-// RUN: %FileCheck -input-file=%t.response %s --check-prefixes CHECK,CHECK-OLD
-// RUN: %sourcekitd-test -req=open -print-raw-response %S/Inputs/syntaxmap-multiple-edits.swift -force-libsyntax-based-processing == -req=edit -print-raw-response -pos=6:13 -length=1 -replace=" " %S/Inputs/syntaxmap-multiple-edits.swift -force-libsyntax-based-processing == -req="edit" -pos=14:1 -length=0 -replace="let y = 2" -print-raw-response %S/Inputs/syntaxmap-multiple-edits.swift -force-libsyntax-based-processing == -req="edit" -pos=8:10 -length=7 -replace='Int64 = 3; let z = 2' -print-raw-response %S/Inputs/syntaxmap-multiple-edits.swift -force-libsyntax-based-processing == -req="edit" -pos=4:9 -length=2 -replace='50 * 95 - 100' -print-raw-response %S/Inputs/syntaxmap-multiple-edits.swift -force-libsyntax-based-processing == -req="edit" -pos=1:1 -length=0 -replace='func firstFunc(x: Int) {}' -print-raw-response %S/Inputs/syntaxmap-multiple-edits.swift -force-libsyntax-based-processing | %sed_clean > %t.libsyntax.response
-// RUN: %FileCheck -input-file=%t.libsyntax.response %s --check-prefixes CHECK,CHECK-NEW
+// RUN: %FileCheck -input-file=%t.response %s
 
 // Initial syntax map
 
@@ -13,29 +11,23 @@
 // CHECK-NEXT:   {
 // CHECK-NEXT:     key.kind: source.lang.swift.syntaxtype.doccomment,
 // CHECK-NEXT:     key.offset: 2,
-// CHECK-OLD-NEXT:     key.length: 14
-// CHECK-NEW-NEXT:     key.length: 13
+// CHECK-NEXT:     key.length: 14
+// CHECK-NEXT:   },
+// CHECK-NEXT:   {
+// CHECK-NEXT:     key.kind: source.lang.swift.syntaxtype.doccomment,
+// CHECK-NEXT:     key.offset: 16,
+// CHECK-NEXT:     key.length: 6
+// CHECK-NEXT:   },
+// CHECK-NEXT:   {
+// CHECK-NEXT:     key.kind: source.lang.swift.syntaxtype.doccomment.field,
+// CHECK-NEXT:     key.offset: 22,
+// CHECK-NEXT:     key.length: 4
+// CHECK-NEXT:   },
+// CHECK-NEXT:   {
+// CHECK-NEXT:     key.kind: source.lang.swift.syntaxtype.doccomment,
+// CHECK-NEXT:     key.offset: 26,
+// CHECK-NEXT:     key.length: 19
 // CHECK-NEXT:   },
-// CHECK-OLD-NEXT:   {
-// CHECK-OLD-NEXT:     key.kind: source.lang.swift.syntaxtype.doccomment,
-// CHECK-OLD-NEXT:     key.offset: 16,
-// CHECK-OLD-NEXT:     key.length: 6
-// CHECK-OLD-NEXT:   },
-// CHECK-OLD-NEXT:   {
-// CHECK-OLD-NEXT:     key.kind: source.lang.swift.syntaxtype.doccomment.field,
-// CHECK-OLD-NEXT:     key.offset: 22,
-// CHECK-OLD-NEXT:     key.length: 4
-// CHECK-OLD-NEXT:   },
-// CHECK-OLD-NEXT:   {
-// CHECK-OLD-NEXT:     key.kind: source.lang.swift.syntaxtype.doccomment,
-// CHECK-OLD-NEXT:     key.offset: 26,
-// CHECK-OLD-NEXT:     key.length: 19
-// CHECK-OLD-NEXT:   },
-// CHECK-NEW-NEXT:   {
-// CHECK-NEW-NEXT:     key.kind: source.lang.swift.syntaxtype.doccomment,
-// CHECK-NEW-NEXT:     key.offset: 16,
-// CHECK-NEW-NEXT:     key.length: 28
-// CHECK-NEW-NEXT:   },
 // CHECK-NEXT:   {
 // CHECK-NEXT:     key.kind: source.lang.swift.syntaxtype.keyword,
 // CHECK-NEXT:     key.offset: 45,

test/SourceKit/SyntaxMapData/syntaxmap-object-literals.swift

@@ -1,7 +1,5 @@
 // RUN: %sourcekitd-test -req=syntax-map %s | %sed_clean > %t.response
 // RUN: diff -u %s.response %t.response
-// RUN: %sourcekitd-test -req=syntax-map %s -force-libsyntax-based-processing | %sed_clean > %t.libsyntax.response
-// RUN: diff -u %s.libsyntax.response %t.libsyntax.response
 
 let image = #imageLiteral(resourceName: "cloud.png")
 let color = #colorLiteral(red: 1, blue: 0, green: 1, alpha: 1)

test/SourceKit/SyntaxMapData/syntaxmap-object-literals.swift.libsyntax.response

@@ -1,132 +0,0 @@
-{
-  key.offset: 0,
-  key.length: 454,
-  key.diagnostic_stage: source.diagnostic.stage.swift.parse,
-  key.syntaxmap: [
-    {
-      key.kind: source.lang.swift.syntaxtype.comment,
-      key.offset: 0,
-      key.length: 70
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.comment,
-      key.offset: 71,
-      key.length: 39
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.comment,
-      key.offset: 111,
-      key.length: 114
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.comment,
-      key.offset: 226,
-      key.length: 59
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.keyword,
-      key.offset: 287,
-      key.length: 3
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.identifier,
-      key.offset: 291,
-      key.length: 5
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.objectliteral,
-      key.offset: 299,
-      key.length: 13
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.identifier,
-      key.offset: 313,
-      key.length: 12
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.string,
-      key.offset: 327,
-      key.length: 11
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.keyword,
-      key.offset: 340,
-      key.length: 3
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.identifier,
-      key.offset: 344,
-      key.length: 5
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.objectliteral,
-      key.offset: 352,
-      key.length: 13
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.identifier,
-      key.offset: 366,
-      key.length: 3
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.number,
-      key.offset: 371,
-      key.length: 1
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.identifier,
-      key.offset: 374,
-      key.length: 4
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.number,
-      key.offset: 380,
-      key.length: 1
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.identifier,
-      key.offset: 383,
-      key.length: 5
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.number,
-      key.offset: 390,
-      key.length: 1
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.identifier,
-      key.offset: 393,
-      key.length: 5
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.number,
-      key.offset: 400,
-      key.length: 1
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.keyword,
-      key.offset: 403,
-      key.length: 3
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.identifier,
-      key.offset: 407,
-      key.length: 4
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.objectliteral,
-      key.offset: 414,
-      key.length: 12
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.identifier,
-      key.offset: 427,
-      key.length: 12
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.string,
-      key.offset: 441,
-      key.length: 10
-    }
-  ]
-}

test/SourceKit/SyntaxMapData/syntaxmap-object-literals.swift.response

@@ -1,6 +1,6 @@
 {
   key.offset: 0,
-  key.length: 454,
+  key.length: 279,
   key.diagnostic_stage: source.diagnostic.stage.swift.parse,
   key.syntaxmap: [
     {
@@ -13,59 +13,49 @@
       key.offset: 71,
       key.length: 40
     },
-    {
-      key.kind: source.lang.swift.syntaxtype.comment,
-      key.offset: 111,
-      key.length: 115
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.comment,
-      key.offset: 226,
-      key.length: 60
-    },
     {
       key.kind: source.lang.swift.syntaxtype.keyword,
-      key.offset: 287,
+      key.offset: 112,
       key.length: 3
     },
     {
       key.kind: source.lang.swift.syntaxtype.identifier,
-      key.offset: 291,
+      key.offset: 116,
       key.length: 5
     },
     {
       key.kind: source.lang.swift.syntaxtype.objectliteral,
-      key.offset: 299,
+      key.offset: 124,
       key.length: 40
     },
     {
       key.kind: source.lang.swift.syntaxtype.keyword,
-      key.offset: 340,
+      key.offset: 165,
       key.length: 3
     },
     {
       key.kind: source.lang.swift.syntaxtype.identifier,
-      key.offset: 344,
+      key.offset: 169,
       key.length: 5
     },
     {
       key.kind: source.lang.swift.syntaxtype.objectliteral,
-      key.offset: 352,
+      key.offset: 177,
       key.length: 50
     },
     {
       key.kind: source.lang.swift.syntaxtype.keyword,
-      key.offset: 403,
+      key.offset: 228,
       key.length: 3
     },
     {
       key.kind: source.lang.swift.syntaxtype.identifier,
-      key.offset: 407,
+      key.offset: 232,
       key.length: 4
     },
     {
       key.kind: source.lang.swift.syntaxtype.objectliteral,
-      key.offset: 414,
+      key.offset: 239,
       key.length: 38
     }
   ]

test/SourceKit/SyntaxMapData/syntaxmap-partial-delete.swift

@@ -1,7 +1,5 @@
 // RUN: %sourcekitd-test -req=open -print-raw-response %S/Inputs/syntaxmap-partial-delete.swift == -req=edit -print-raw-response -pos=2:10 -length=2 -replace='' %S/Inputs/syntaxmap-partial-delete.swift | %sed_clean > %t.response
 // RUN: %FileCheck -input-file=%t.response %s
-// RUN: %sourcekitd-test -req=open -print-raw-response %S/Inputs/syntaxmap-partial-delete.swift -force-libsyntax-based-processing == -req=edit -print-raw-response -pos=2:10 -length=2 -replace='' %S/Inputs/syntaxmap-partial-delete.swift -force-libsyntax-based-processing | %sed_clean > %t.libsyntax.response
-// RUN: %FileCheck -input-file=%t.libsyntax.response %s
 
 // CHECK: {{^}}{
 // CHECK-NEXT: key.offset: 0,

test/SourceKit/SyntaxMapData/syntaxmap-pound-keyword.swift

@@ -1,7 +1,5 @@
 // RUN: %sourcekitd-test -req=syntax-map %s | %sed_clean > %t.response
 // RUN: diff -u %s.response %t.response
-// RUN: %sourcekitd-test -req=syntax-map %s -force-libsyntax-based-processing | %sed_clean > %t.libsyntax.response
-// RUN: diff -u %s.libsyntax.response %t.libsyntax.response
 
 let fn = #function
 let f = #file

test/SourceKit/SyntaxMapData/syntaxmap-pound-keyword.swift.libsyntax.response

@@ -1,187 +0,0 @@
-{
-  key.offset: 0,
-  key.length: 517,
-  key.diagnostic_stage: source.diagnostic.stage.swift.parse,
-  key.syntaxmap: [
-    {
-      key.kind: source.lang.swift.syntaxtype.comment,
-      key.offset: 0,
-      key.length: 70
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.comment,
-      key.offset: 71,
-      key.length: 39
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.comment,
-      key.offset: 111,
-      key.length: 114
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.comment,
-      key.offset: 226,
-      key.length: 59
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.keyword,
-      key.offset: 287,
-      key.length: 3
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.identifier,
-      key.offset: 291,
-      key.length: 2
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.keyword,
-      key.offset: 296,
-      key.length: 9
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.keyword,
-      key.offset: 306,
-      key.length: 3
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.identifier,
-      key.offset: 310,
-      key.length: 1
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.keyword,
-      key.offset: 314,
-      key.length: 5
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.keyword,
-      key.offset: 320,
-      key.length: 3
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.identifier,
-      key.offset: 324,
-      key.length: 1
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.keyword,
-      key.offset: 328,
-      key.length: 5
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.keyword,
-      key.offset: 334,
-      key.length: 3
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.identifier,
-      key.offset: 338,
-      key.length: 1
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.keyword,
-      key.offset: 342,
-      key.length: 7
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.keyword,
-      key.offset: 351,
-      key.length: 2
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.keyword,
-      key.offset: 354,
-      key.length: 10
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.keyword,
-      key.offset: 365,
-      key.length: 3
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.number,
-      key.offset: 369,
-      key.length: 3
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.buildconfig.keyword,
-      key.offset: 381,
-      key.length: 3
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.keyword,
-      key.offset: 385,
-      key.length: 5
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.pounddirective.keyword,
-      key.offset: 391,
-      key.length: 6
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.string,
-      key.offset: 398,
-      key.length: 7
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.buildconfig.keyword,
-      key.offset: 407,
-      key.length: 7
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.keyword,
-      key.offset: 415,
-      key.length: 4
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.pounddirective.keyword,
-      key.offset: 420,
-      key.length: 8
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.string,
-      key.offset: 429,
-      key.length: 9
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.buildconfig.keyword,
-      key.offset: 440,
-      key.length: 5
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.pounddirective.keyword,
-      key.offset: 446,
-      key.length: 15
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.identifier,
-      key.offset: 462,
-      key.length: 4
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.string,
-      key.offset: 468,
-      key.length: 12
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.identifier,
-      key.offset: 482,
-      key.length: 4
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.number,
-      key.offset: 487,
-      key.length: 3
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.pounddirective.keyword,
-      key.offset: 492,
-      key.length: 15
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.buildconfig.keyword,
-      key.offset: 510,
-      key.length: 6
-    }
-  ]
-}

test/SourceKit/SyntaxMapData/syntaxmap-pound-keyword.swift.response

@@ -1,6 +1,6 @@
 {
   key.offset: 0,
-  key.length: 517,
+  key.length: 342,
   key.diagnostic_stage: source.diagnostic.stage.swift.parse,
   key.syntaxmap: [
     {
@@ -13,174 +13,164 @@
       key.offset: 71,
       key.length: 40
     },
-    {
-      key.kind: source.lang.swift.syntaxtype.comment,
-      key.offset: 111,
-      key.length: 115
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.comment,
-      key.offset: 226,
-      key.length: 60
-    },
     {
       key.kind: source.lang.swift.syntaxtype.keyword,
-      key.offset: 287,
+      key.offset: 112,
       key.length: 3
     },
     {
       key.kind: source.lang.swift.syntaxtype.identifier,
-      key.offset: 291,
+      key.offset: 116,
       key.length: 2
     },
     {
       key.kind: source.lang.swift.syntaxtype.keyword,
-      key.offset: 296,
+      key.offset: 121,
       key.length: 9
     },
     {
       key.kind: source.lang.swift.syntaxtype.keyword,
-      key.offset: 306,
+      key.offset: 131,
       key.length: 3
     },
     {
       key.kind: source.lang.swift.syntaxtype.identifier,
-      key.offset: 310,
+      key.offset: 135,
       key.length: 1
     },
     {
       key.kind: source.lang.swift.syntaxtype.keyword,
-      key.offset: 314,
+      key.offset: 139,
       key.length: 5
     },
     {
       key.kind: source.lang.swift.syntaxtype.keyword,
-      key.offset: 320,
+      key.offset: 145,
       key.length: 3
     },
     {
       key.kind: source.lang.swift.syntaxtype.identifier,
-      key.offset: 324,
+      key.offset: 149,
       key.length: 1
     },
     {
       key.kind: source.lang.swift.syntaxtype.keyword,
-      key.offset: 328,
+      key.offset: 153,
       key.length: 5
     },
     {
       key.kind: source.lang.swift.syntaxtype.keyword,
-      key.offset: 334,
+      key.offset: 159,
       key.length: 3
     },
     {
       key.kind: source.lang.swift.syntaxtype.identifier,
-      key.offset: 338,
+      key.offset: 163,
       key.length: 1
     },
     {
       key.kind: source.lang.swift.syntaxtype.keyword,
-      key.offset: 342,
+      key.offset: 167,
       key.length: 7
     },
     {
       key.kind: source.lang.swift.syntaxtype.keyword,
-      key.offset: 351,
+      key.offset: 176,
       key.length: 2
     },
     {
       key.kind: source.lang.swift.syntaxtype.keyword,
-      key.offset: 354,
+      key.offset: 179,
       key.length: 10
     },
     {
       key.kind: source.lang.swift.syntaxtype.keyword,
-      key.offset: 365,
+      key.offset: 190,
       key.length: 3
     },
     {
       key.kind: source.lang.swift.syntaxtype.number,
-      key.offset: 369,
+      key.offset: 194,
       key.length: 3
     },
     {
       key.kind: source.lang.swift.syntaxtype.buildconfig.keyword,
-      key.offset: 381,
+      key.offset: 206,
       key.length: 3
     },
     {
       key.kind: source.lang.swift.syntaxtype.keyword,
-      key.offset: 385,
+      key.offset: 210,
       key.length: 5
     },
     {
       key.kind: source.lang.swift.syntaxtype.pounddirective.keyword,
-      key.offset: 391,
+      key.offset: 216,
       key.length: 6
     },
     {
       key.kind: source.lang.swift.syntaxtype.string,
-      key.offset: 398,
+      key.offset: 223,
       key.length: 7
     },
     {
       key.kind: source.lang.swift.syntaxtype.buildconfig.keyword,
-      key.offset: 407,
+      key.offset: 232,
       key.length: 7
     },
     {
       key.kind: source.lang.swift.syntaxtype.keyword,
-      key.offset: 415,
+      key.offset: 240,
       key.length: 4
     },
     {
       key.kind: source.lang.swift.syntaxtype.pounddirective.keyword,
-      key.offset: 420,
+      key.offset: 245,
       key.length: 8
     },
     {
       key.kind: source.lang.swift.syntaxtype.string,
-      key.offset: 429,
+      key.offset: 254,
       key.length: 9
     },
     {
       key.kind: source.lang.swift.syntaxtype.buildconfig.keyword,
-      key.offset: 440,
+      key.offset: 265,
       key.length: 5
     },
     {
       key.kind: source.lang.swift.syntaxtype.pounddirective.keyword,
-      key.offset: 446,
+      key.offset: 271,
       key.length: 15
     },
     {
       key.kind: source.lang.swift.syntaxtype.identifier,
-      key.offset: 462,
+      key.offset: 287,
       key.length: 4
     },
     {
       key.kind: source.lang.swift.syntaxtype.string,
-      key.offset: 468,
+      key.offset: 293,
       key.length: 12
     },
     {
       key.kind: source.lang.swift.syntaxtype.identifier,
-      key.offset: 482,
+      key.offset: 307,
       key.length: 4
     },
     {
       key.kind: source.lang.swift.syntaxtype.number,
-      key.offset: 487,
+      key.offset: 312,
       key.length: 3
     },
     {
       key.kind: source.lang.swift.syntaxtype.pounddirective.keyword,
-      key.offset: 492,
+      key.offset: 317,
       key.length: 15
     },
     {
       key.kind: source.lang.swift.syntaxtype.buildconfig.keyword,
-      key.offset: 510,
+      key.offset: 335,
       key.length: 6
     }
   ]

test/SourceKit/SyntaxMapData/syntaxmap.swift

@@ -1,5 +1,3 @@
 // XFAIL: broken_std_regex
 // RUN: %sourcekitd-test -req=syntax-map %S/Inputs/syntaxmap.swift > %t.response
 // RUN: diff -u %s.response %t.response
-// RUN: %sourcekitd-test -req=syntax-map %S/Inputs/syntaxmap.swift -force-libsyntax-based-processing > %t.libsyntax.response
-// RUN: diff -u %s.libsyntax.response %t.libsyntax.response

test/SourceKit/SyntaxMapData/syntaxmap.swift.libsyntax.response

@@ -1,467 +0,0 @@
-{
-  key.offset: 0,
-  key.length: 1049,
-  key.diagnostic_stage: source.diagnostic.stage.swift.parse,
-  key.syntaxmap: [
-    {
-      key.kind: source.lang.swift.syntaxtype.comment,
-      key.offset: 0,
-      key.length: 21
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.comment,
-      key.offset: 22,
-      key.length: 59
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.keyword,
-      key.offset: 83,
-      key.length: 5
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.identifier,
-      key.offset: 89,
-      key.length: 3
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.keyword,
-      key.offset: 99,
-      key.length: 3
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.identifier,
-      key.offset: 103,
-      key.length: 4
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.typeidentifier,
-      key.offset: 110,
-      key.length: 3
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.keyword,
-      key.offset: 118,
-      key.length: 3
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.identifier,
-      key.offset: 122,
-      key.length: 1
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.identifier,
-      key.offset: 125,
-      key.length: 1
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.typeidentifier,
-      key.offset: 129,
-      key.length: 3
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.buildconfig.keyword,
-      key.offset: 136,
-      key.length: 3
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.buildconfig.id,
-      key.offset: 140,
-      key.length: 4
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.keyword,
-      key.offset: 145,
-      key.length: 3
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.identifier,
-      key.offset: 149,
-      key.length: 1
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.typeidentifier,
-      key.offset: 153,
-      key.length: 3
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.buildconfig.keyword,
-      key.offset: 157,
-      key.length: 5
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.keyword,
-      key.offset: 163,
-      key.length: 3
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.identifier,
-      key.offset: 167,
-      key.length: 1
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.typeidentifier,
-      key.offset: 171,
-      key.length: 5
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.buildconfig.keyword,
-      key.offset: 177,
-      key.length: 6
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.comment,
-      key.offset: 185,
-      key.length: 26
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.comment,
-      key.offset: 212,
-      key.length: 48
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.comment,
-      key.offset: 261,
-      key.length: 22
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.comment,
-      key.offset: 284,
-      key.length: 23
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.keyword,
-      key.offset: 309,
-      key.length: 3
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.identifier,
-      key.offset: 313,
-      key.length: 3
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.string,
-      key.offset: 319,
-      key.length: 1
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.string,
-      key.offset: 320,
-      key.length: 12
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.string_interpolation_anchor,
-      key.offset: 333,
-      key.length: 1
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.number,
-      key.offset: 334,
-      key.length: 1
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.string_interpolation_anchor,
-      key.offset: 335,
-      key.length: 1
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.string,
-      key.offset: 336,
-      key.length: 1
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.keyword,
-      key.offset: 339,
-      key.length: 4
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.placeholder,
-      key.offset: 344,
-      key.length: 9
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.doccomment,
-      key.offset: 360,
-      key.length: 22
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.doccomment,
-      key.offset: 383,
-      key.length: 3
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.doccomment,
-      key.offset: 387,
-      key.length: 15
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.doccomment,
-      key.offset: 403,
-      key.length: 3
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.doccomment,
-      key.offset: 407,
-      key.length: 17
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.doccomment,
-      key.offset: 425,
-      key.length: 14
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.doccomment,
-      key.offset: 440,
-      key.length: 7
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.doccomment,
-      key.offset: 448,
-      key.length: 32
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.doccomment,
-      key.offset: 481,
-      key.length: 3
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.doccomment,
-      key.offset: 485,
-      key.length: 27
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.doccomment,
-      key.offset: 513,
-      key.length: 22
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.keyword,
-      key.offset: 536,
-      key.length: 4
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.identifier,
-      key.offset: 541,
-      key.length: 3
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.identifier,
-      key.offset: 545,
-      key.length: 1
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.typeidentifier,
-      key.offset: 548,
-      key.length: 3
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.identifier,
-      key.offset: 553,
-      key.length: 1
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.typeidentifier,
-      key.offset: 556,
-      key.length: 3
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.typeidentifier,
-      key.offset: 564,
-      key.length: 3
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.keyword,
-      key.offset: 570,
-      key.length: 6
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.identifier,
-      key.offset: 577,
-      key.length: 1
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.identifier,
-      key.offset: 581,
-      key.length: 1
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.doccomment,
-      key.offset: 586,
-      key.length: 139
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.keyword,
-      key.offset: 726,
-      key.length: 4
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.identifier,
-      key.offset: 731,
-      key.length: 3
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.comment,
-      key.offset: 741,
-      key.length: 30
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.comment,
-      key.offset: 772,
-      key.length: 16
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.comment,
-      key.offset: 789,
-      key.length: 23
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.comment,
-      key.offset: 813,
-      key.length: 35
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.keyword,
-      key.offset: 850,
-      key.length: 1
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.number,
-      key.offset: 855,
-      key.length: 3
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.keyword,
-      key.offset: 860,
-      key.length: 4
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.identifier,
-      key.offset: 865,
-      key.length: 18
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.identifier,
-      key.offset: 884,
-      key.length: 2
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.identifier,
-      key.offset: 887,
-      key.length: 5
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.typeidentifier,
-      key.offset: 894,
-      key.length: 3
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.keyword,
-      key.offset: 899,
-      key.length: 1
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.identifier,
-      key.offset: 901,
-      key.length: 4
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.keyword,
-      key.offset: 908,
-      key.length: 1
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.identifier,
-      key.offset: 910,
-      key.length: 7
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.typeidentifier,
-      key.offset: 919,
-      key.length: 3
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.typeidentifier,
-      key.offset: 927,
-      key.length: 4
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.identifier,
-      key.offset: 937,
-      key.length: 2
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.typeidentifier,
-      key.offset: 941,
-      key.length: 3
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.typeidentifier,
-      key.offset: 946,
-      key.length: 6
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.keyword,
-      key.offset: 958,
-      key.length: 3
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.identifier,
-      key.offset: 962,
-      key.length: 6
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.identifier,
-      key.offset: 971,
-      key.length: 2
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.typeidentifier,
-      key.offset: 975,
-      key.length: 3
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.typeidentifier,
-      key.offset: 980,
-      key.length: 6
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.number,
-      key.offset: 991,
-      key.length: 1
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.string,
-      key.offset: 994,
-      key.length: 6
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.keyword,
-      key.offset: 1004,
-      key.length: 6
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.identifier,
-      key.offset: 1011,
-      key.length: 9
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.identifier,
-      key.offset: 1023,
-      key.length: 6
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.identifier,
-      key.offset: 1033,
-      key.length: 2
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.number,
-      key.offset: 1037,
-      key.length: 1
-    },
-    {
-      key.kind: source.lang.swift.syntaxtype.string,
-      key.offset: 1040,
-      key.length: 5
-    }
-  ]
-}

tools/SourceKit/include/SourceKit/Core/LangSupport.h

@@ -213,9 +213,12 @@ public:
 
   virtual void handleRequestError(const char *Description) = 0;
 
+  virtual bool syntaxMapEnabled() = 0;
   virtual bool handleSyntaxMap(unsigned Offset, unsigned Length,
                                UIdent Kind) = 0;
 
+  virtual bool documentStructureEnabled() = 0;
+
   virtual bool handleSemanticAnnotation(unsigned Offset, unsigned Length,
                                         UIdent Kind, bool isSystem) = 0;
 
@@ -264,10 +267,6 @@ public:
       std::vector<SourceFileRange> ReuseRegions) = 0;
 
   virtual void finished() {}
-
-  // FIXME: This is just for bootstrapping incremental syntax tree parsing.
-  // Remove it once when we are able to incrementally transfer the syntax tree
-  virtual bool forceLibSyntaxBasedProcessing() = 0;
 };
 
 class OptionsDictionary {

tools/SourceKit/lib/SwiftLang/SwiftEditor.cpp

@@ -37,7 +37,6 @@
 #include "swift/IDE/SyntaxModel.h"
 #include "swift/Subsystems.h"
 #include "swift/Syntax/Serialization/SyntaxSerialization.h"
-#include "swift/Syntax/SyntaxClassifier.h"
 #include "swift/Syntax/SyntaxNodes.h"
 
 #include "llvm/Support/ErrorHandling.h"
@@ -559,122 +558,6 @@ struct SwiftSyntaxMap {
   }
 };
 
-class SyntaxToSyntaxMapConverter : public SyntaxVisitor {
-  SwiftSyntaxMap &SyntaxMap;
-
-  std::map<SyntaxNodeId, SyntaxClassification> TokenClassifications;
-
-public:
-  SyntaxToSyntaxMapConverter(
-      SwiftSyntaxMap &SyntaxMap,
-      std::map<SyntaxNodeId, SyntaxClassification> TokenClassifications)
-      : SyntaxMap(SyntaxMap), TokenClassifications(TokenClassifications) {}
-
-private:
-  void visitTrivia(Trivia Trivia, unsigned Offset) {
-    for (auto TriviaPiece : Trivia) {
-      visitTriviaPiece(TriviaPiece, Offset);
-      Offset += TriviaPiece.getTextLength();
-    }
-  }
-
-  void visitTriviaPiece(TriviaPiece TriviaPiece, unsigned Offset) {
-    llvm::Optional<SyntaxNodeKind> Kind;
-    switch (TriviaPiece.getKind()) {
-    case TriviaKind::Space:
-    case TriviaKind::Tab:
-    case TriviaKind::VerticalTab:
-    case TriviaKind::Formfeed:
-    case TriviaKind::Newline:
-    case TriviaKind::CarriageReturn:
-    case TriviaKind::CarriageReturnLineFeed:
-    case swift::syntax::TriviaKind::Backtick:
-    case TriviaKind::GarbageText:
-      Kind = llvm::None;
-      break;
-    case swift::syntax::TriviaKind::LineComment:
-      Kind = SyntaxNodeKind::CommentLine;
-      break;
-    case TriviaKind::BlockComment:
-      Kind = SyntaxNodeKind::CommentBlock;
-      break;
-    case TriviaKind::DocLineComment:
-      Kind = SyntaxNodeKind::DocCommentLine;
-      break;
-    case TriviaKind::DocBlockComment:
-      Kind = SyntaxNodeKind::DocCommentBlock;
-      break;
-    }
-    if (Kind.hasValue()) {
-      SwiftSyntaxToken Token(Offset, TriviaPiece.getTextLength(),
-                             Kind.getValue());
-      SyntaxMap.addToken(Token);
-    }
-  }
-
-  llvm::Optional<SyntaxNodeKind>
-  getKindForSyntaxClassification(SyntaxClassification Classification) const {
-    // FIXME: We should really use the same enum so that the conversion is not
-    // necessary
-    switch (Classification) {
-    case SyntaxClassification::None:
-      return llvm::None;
-    case SyntaxClassification::Keyword:
-      return SyntaxNodeKind::Keyword;
-    case SyntaxClassification::Identifier:
-      return SyntaxNodeKind::Identifier;
-    case SyntaxClassification::DollarIdentifier:
-      return SyntaxNodeKind::DollarIdent;
-    case SyntaxClassification::IntegerLiteral:
-      return SyntaxNodeKind::Integer;
-    case SyntaxClassification::FloatingLiteral:
-      return SyntaxNodeKind::Floating;
-    case SyntaxClassification::StringLiteral:
-      return SyntaxNodeKind::String;
-    case SyntaxClassification::StringInterpolationAnchor:
-      return SyntaxNodeKind::StringInterpolationAnchor;
-    case SyntaxClassification::TypeIdentifier:
-      return SyntaxNodeKind::TypeId;
-    case SyntaxClassification::BuildConfigKeyword:
-      return SyntaxNodeKind::BuildConfigKeyword;
-    case SyntaxClassification::BuildConfigId:
-      return SyntaxNodeKind::BuildConfigId;
-    case SyntaxClassification::PoundDirectiveKeyword:
-      return SyntaxNodeKind::PoundDirectiveKeyword;
-    case SyntaxClassification::Attribute:
-      return SyntaxNodeKind::AttributeBuiltin;
-    case SyntaxClassification::EditorPlaceholder:
-      return SyntaxNodeKind::EditorPlaceholder;
-    case SyntaxClassification::ObjectLiteral:
-      return SyntaxNodeKind::ObjectLiteral;
-    }
-  }
-
-  virtual void visit(TokenSyntax Token) override {
-    if (Token.isMissing())
-      return;
-
-    auto LeadingTriviaOffset =
-        Token.getAbsolutePositionBeforeLeadingTrivia().getOffset();
-    visitTrivia(Token.getLeadingTrivia(), LeadingTriviaOffset);
-
-    SyntaxClassification Classification = TokenClassifications[Token.getId()];
-    auto Kind = getKindForSyntaxClassification(Classification);
-    unsigned TokenStart = Token.getAbsolutePosition().getOffset();
-    unsigned TokenLength = Token.getRaw()->getTokenText().size();
-    if (Kind.hasValue() && TokenLength > 0) {
-      SwiftSyntaxToken Token(TokenStart, TokenLength, Kind.getValue());
-      SyntaxMap.addToken(Token);
-    }
-
-    auto TrailingTriviaOffset = TokenStart + TokenLength;
-    visitTrivia(Token.getTrailingTrivia(), TrailingTriviaOffset);
-  }
-
-public:
-  void writeToSyntaxMap(Syntax Node) { Node.accept(*this); }
-};
-
 struct EditorConsumerSyntaxMapEntry {
   unsigned Offset;
   unsigned Length;
@@ -1918,14 +1801,21 @@ void SwiftEditorDocument::readSyntaxInfo(EditorConsumer &Consumer) {
 
   SwiftSyntaxMap NewMap = SwiftSyntaxMap(Impl.SyntaxMap.Tokens.size() + 16);
 
-  if (Consumer.forceLibSyntaxBasedProcessing()) {
+  if (Consumer.syntaxTreeEnabled()) {
     auto SyntaxTree = Impl.SyntaxInfo->getSourceFile().getSyntaxRoot();
-
-    SyntaxClassifier Classifier;
-    auto Classification = Classifier.classify(SyntaxTree);
-    SyntaxToSyntaxMapConverter Printer(NewMap, Classification);
-    Printer.writeToSyntaxMap(SyntaxTree);
     Impl.SyntaxTree.emplace(SyntaxTree);
+    if (Consumer.syntaxMapEnabled()) {
+      Consumer.handleRequestError(
+          "Retrieving both a syntax map and a syntax tree at the same time is "
+          "not supported. Use the SyntaxClassifier in swiftSyntax to generate "
+          "the syntax map on the Swift side.");
+    }
+    if (Consumer.documentStructureEnabled()) {
+      Consumer.handleRequestError(
+          "Retrieving both the document structure and a syntax tree at the "
+          "same time is not supported. Use the syntax tree to compute the "
+          "document structure.");
+    }
   } else {
     ide::SyntaxModelContext ModelContext(Impl.SyntaxInfo->getSourceFile());
 
@@ -1933,28 +1823,28 @@ void SwiftEditorDocument::readSyntaxInfo(EditorConsumer &Consumer) {
         NewMap, Impl.SyntaxInfo->getSourceManager(), Consumer,
         Impl.SyntaxInfo->getBufferID());
     ModelContext.walk(SyntaxWalker);
-  }
 
-  bool SawChanges = true;
-  if (Impl.Edited) {
-    // We're ansering an edit request. Report all highlighted token ranges not
-    // in the previous syntax map to the Consumer and extend the AffectedRange
-    // to contain all added/removed token ranges.
-    SawChanges = NewMap.forEachChanged(Impl.SyntaxMap, Impl.AffectedRange,
-                                       Consumer);
-  } else {
-    // The is an open/initialise. Report all highlighted token ranges to the
-    // Consumer.
-    NewMap.forEach(Consumer);
-  }
-  Impl.SyntaxMap = std::move(NewMap);
+    bool SawChanges = true;
+    if (Impl.Edited) {
+      // We're ansering an edit request. Report all highlighted token ranges not
+      // in the previous syntax map to the Consumer and extend the AffectedRange
+      // to contain all added/removed token ranges.
+      SawChanges =
+          NewMap.forEachChanged(Impl.SyntaxMap, Impl.AffectedRange, Consumer);
+    } else {
+      // The is an open/initialise. Report all highlighted token ranges to the
+      // Consumer.
+      NewMap.forEach(Consumer);
+    }
+    Impl.SyntaxMap = std::move(NewMap);
 
-  // Recording an affected length of 0 still results in the client updating its
-  // copy of the syntax map (by clearning all tokens on the line of the affected
-  // offset). We need to not record it at all to signal a no-op.
-  if (SawChanges)
-    Consumer.recordAffectedRange(Impl.AffectedRange->Offset,
-                                 Impl.AffectedRange->length());
+    // Recording an affected length of 0 still results in the client updating
+    // its copy of the syntax map (by clearning all tokens on the line of the
+    // affected offset). We need to not record it at all to signal a no-op.
+    if (SawChanges)
+      Consumer.recordAffectedRange(Impl.AffectedRange->Offset,
+                                   Impl.AffectedRange->length());
+  }
 }
 
 void SwiftEditorDocument::readSemanticInfo(ImmutableTextSnapshotRef Snapshot,
@@ -2189,13 +2079,11 @@ void SwiftLangSupport::editorOpen(StringRef Name, llvm::MemoryBuffer *Buf,
                                   ArrayRef<const char *> Args) {
 
   ImmutableTextSnapshotRef Snapshot = nullptr;
-  const bool BuildSyntaxTree =
-      Consumer.syntaxTreeEnabled() || Consumer.forceLibSyntaxBasedProcessing();
   auto EditorDoc = EditorDocuments.getByUnresolvedName(Name);
   if (!EditorDoc) {
     EditorDoc = new SwiftEditorDocument(Name, *this);
     Snapshot = EditorDoc->initializeText(Buf, Args);
-    EditorDoc->parse(Snapshot, *this, BuildSyntaxTree);
+    EditorDoc->parse(Snapshot, *this, Consumer.syntaxTreeEnabled());
     if (EditorDocuments.getOrUpdate(Name, *this, EditorDoc)) {
       // Document already exists, re-initialize it. This should only happen
       // if we get OPEN request while the previous document is not closed.
@@ -2208,7 +2096,7 @@ void SwiftLangSupport::editorOpen(StringRef Name, llvm::MemoryBuffer *Buf,
 
   if (!Snapshot) {
     Snapshot = EditorDoc->initializeText(Buf, Args);
-    EditorDoc->parse(Snapshot, *this, BuildSyntaxTree);
+    EditorDoc->parse(Snapshot, *this, Consumer.syntaxTreeEnabled());
   }
 
   if (Consumer.needsSemanticInfo()) {
@@ -2352,14 +2240,9 @@ void SwiftLangSupport::editorReplaceText(StringRef Name,
     Snapshot = EditorDoc->replaceText(Offset, Length, Buf,
                                       Consumer.needsSemanticInfo());
     assert(Snapshot);
-    bool BuildSyntaxTree = Consumer.syntaxTreeEnabled() ||
-                           Consumer.forceLibSyntaxBasedProcessing();
 
     llvm::Optional<SyntaxParsingCache> SyntaxCache = llvm::None;
-    bool IncrementalSyntaxParsing = Consumer.forceLibSyntaxBasedProcessing() ||
-                                    Consumer.syntaxTreeTransferMode() ==
-                                        SyntaxTreeTransferMode::Incremental;
-    if (EditorDoc->getSyntaxTree().hasValue() && IncrementalSyntaxParsing) {
+    if (EditorDoc->getSyntaxTree().hasValue()) {
       SyntaxCache.emplace(EditorDoc->getSyntaxTree().getValue());
       SyntaxCache->addEdit(Offset, Offset + Length, Buf->getBufferSize());
     }
@@ -2368,7 +2251,8 @@ void SwiftLangSupport::editorReplaceText(StringRef Name,
     if (SyntaxCache.hasValue()) {
       SyntaxCachePtr = SyntaxCache.getPointer();
     }
-    EditorDoc->parse(Snapshot, *this, BuildSyntaxTree, SyntaxCachePtr);
+    EditorDoc->parse(Snapshot, *this, Consumer.syntaxTreeEnabled(),
+                     SyntaxCachePtr);
     EditorDoc->readSyntaxInfo(Consumer);
 
     // Log reuse information

tools/SourceKit/tools/sourcekitd-test/Options.td

@@ -127,9 +127,6 @@ def cancel_on_subsequent_request : Separate<["-"], "cancel-on-subsequent-request
 def cancel_on_subsequent_request_EQ : Joined<["-"], "cancel-on-subsequent-request=">,
   Alias<cancel_on_subsequent_request>;
 
-def force_libsyntax_based_processing : Flag<["-"], "force-libsyntax-based-processing">,
-  HelpText<"Use libSyntax to process syntactic information">;
-
 def time_request : Flag<["-"], "time-request">,
   HelpText<"Print the time taken to process the request">;
 

tools/SourceKit/tools/sourcekitd-test/TestOptions.cpp

@@ -349,10 +349,6 @@ bool TestOptions::parseArgs(llvm::ArrayRef<const char *> Args) {
       }
       break;
 
-    case OPT_force_libsyntax_based_processing:
-      ForceLibSyntaxBasedProcessing = true;
-      break;
-
     case OPT_UNKNOWN:
       llvm::errs() << "error: unknown argument: "
                    << InputArg->getAsString(ParsedArgs) << '\n'

tools/SourceKit/tools/sourcekitd-test/TestOptions.h

@@ -107,7 +107,6 @@ struct TestOptions {
   bool timeRequest = false;
   unsigned repeatRequest = 1;
   llvm::Optional<bool> CancelOnSubsequentRequest;
-  bool ForceLibSyntaxBasedProcessing = false;
   bool parseArgs(llvm::ArrayRef<const char *> Args);
   void printHelp(bool ShowHidden) const;
 };

tools/SourceKit/tools/sourcekitd-test/sourcekitd-test.cpp

@@ -648,9 +648,6 @@ static int handleTestInvocation(TestOptions Opts, TestOptions &InitOpts) {
     sourcekitd_request_dictionary_set_int64(Req, KeyEnableStructure, false);
     sourcekitd_request_dictionary_set_uid(Req, KeySyntaxTreeTransferMode,
                                           KindSyntaxTreeOff);
-    sourcekitd_request_dictionary_set_int64(Req,
-                                            KeyForceLibSyntaxBasedProcessing,
-                                            Opts.ForceLibSyntaxBasedProcessing);
     sourcekitd_request_dictionary_set_int64(Req, KeySyntacticOnly, !Opts.UsedSema);
     break;
 
@@ -705,9 +702,6 @@ static int handleTestInvocation(TestOptions Opts, TestOptions &InitOpts) {
   case SourceKitRequest::Open:
     sourcekitd_request_dictionary_set_uid(Req, KeyRequest, RequestEditorOpen);
     sourcekitd_request_dictionary_set_string(Req, KeyName, SemaName.c_str());
-    sourcekitd_request_dictionary_set_int64(Req,
-                                            KeyForceLibSyntaxBasedProcessing,
-                                            Opts.ForceLibSyntaxBasedProcessing);
     break;
 
   case SourceKitRequest::Close:
@@ -723,9 +717,6 @@ static int handleTestInvocation(TestOptions Opts, TestOptions &InitOpts) {
     sourcekitd_request_dictionary_set_int64(Req, KeyLength, Opts.Length);
     sourcekitd_request_dictionary_set_string(Req, KeySourceText,
                                        Opts.ReplaceText.getValue().c_str());
-    sourcekitd_request_dictionary_set_int64(Req,
-                                            KeyForceLibSyntaxBasedProcessing,
-                                            Opts.ForceLibSyntaxBasedProcessing);
     break;
 
   case SourceKitRequest::PrintAnnotations:
@@ -1074,9 +1065,6 @@ static bool handleResponse(sourcekitd_response_t Resp, const TestOptions &Opts,
                                                 EnableSubStructure);
         sourcekitd_request_dictionary_set_int64(EdReq, KeySyntacticOnly,
                                                 !Opts.UsedSema);
-        sourcekitd_request_dictionary_set_int64(
-            EdReq, KeyForceLibSyntaxBasedProcessing,
-            Opts.ForceLibSyntaxBasedProcessing);
 
         sourcekitd_response_t EdResp = sendRequestSync(EdReq, Opts);
         sourcekitd_response_description_dump_filedesc(EdResp, STDOUT_FILENO);

tools/SourceKit/tools/sourcekitd/lib/API/Requests.cpp

@@ -76,9 +76,6 @@ struct SKEditorConsumerOptions {
   SyntaxTreeTransferMode SyntaxTransferMode = SyntaxTreeTransferMode::Off;
   bool SyntacticOnly = false;
   bool EnableSyntaxReuseInfo = false;
-  // FIXME: This is just for bootstrapping incremental syntax tree parsing.
-  // Remove it once when we are able to incrementally transfer the syntax tree
-  bool ForceLibSyntaxBasedProcessing = false;
 };
 
 } // anonymous namespace
@@ -453,9 +450,6 @@ void handleRequestImpl(sourcekitd_object_t ReqObj, ResponseReceiver Rec) {
     auto TransferModeUID = Req.getUID(KeySyntaxTreeTransferMode);
     int64_t SyntacticOnly = false;
     Req.getInt64(KeySyntacticOnly, SyntacticOnly, /*isOptional=*/true);
-    int64_t ForceLibSyntaxBasedProcessing = false;
-    Req.getInt64(KeyForceLibSyntaxBasedProcessing,
-                 ForceLibSyntaxBasedProcessing, /*isOptional=*/true);
     int64_t EnableSyntaxReuseInfo = false;
     Req.getInt64(KeyEnableSyntaxReuseRegions, EnableSyntaxReuseInfo,
                  /*isOptional=*/true);
@@ -467,7 +461,6 @@ void handleRequestImpl(sourcekitd_object_t ReqObj, ResponseReceiver Rec) {
     Opts.SyntaxTransferMode = syntaxTransferModeFromUID(TransferModeUID);
     Opts.SyntacticOnly = SyntacticOnly;
     Opts.EnableSyntaxReuseInfo = EnableSyntaxReuseInfo;
-    Opts.ForceLibSyntaxBasedProcessing = ForceLibSyntaxBasedProcessing;
     return Rec(editorOpen(*Name, InputBuf.get(), Opts, Args));
   }
   if (ReqUID == RequestEditorClose) {
@@ -501,10 +494,6 @@ void handleRequestImpl(sourcekitd_object_t ReqObj, ResponseReceiver Rec) {
     auto TransferModeUID = Req.getUID(KeySyntaxTreeTransferMode);
     int64_t SyntacticOnly = false;
     Req.getInt64(KeySyntacticOnly, SyntacticOnly, /*isOptional=*/true);
-    int64_t ForceLibSyntaxBasedProcessing = false;
-    Req.getInt64(KeyForceLibSyntaxBasedProcessing,
-                 ForceLibSyntaxBasedProcessing,
-                 /*isOptional=*/true);
     int64_t EnableSyntaxReuseInfo = false;
     Req.getInt64(KeyEnableSyntaxReuseRegions, EnableSyntaxReuseInfo,
                  /*isOptional=*/true);
@@ -517,7 +506,6 @@ void handleRequestImpl(sourcekitd_object_t ReqObj, ResponseReceiver Rec) {
     Opts.EnableSyntaxReuseInfo = EnableSyntaxReuseInfo;
     Opts.SyntacticOnly = SyntacticOnly;
     Opts.EnableSyntaxReuseInfo = EnableSyntaxReuseInfo;
-    Opts.ForceLibSyntaxBasedProcessing = ForceLibSyntaxBasedProcessing;
 
     return Rec(editorReplaceText(*Name, InputBuf.get(), Offset, Length, Opts));
   }
@@ -2055,11 +2043,15 @@ public:
 
   void handleRequestError(const char *Description) override;
 
+  bool syntaxMapEnabled() override { return Opts.EnableSyntaxMap; }
+
   bool handleSyntaxMap(unsigned Offset, unsigned Length, UIdent Kind) override;
 
   bool handleSemanticAnnotation(unsigned Offset, unsigned Length, UIdent Kind,
                                 bool isSystem) override;
 
+  bool documentStructureEnabled() override { return Opts.EnableStructure; }
+
   bool beginDocumentSubStructure(unsigned Offset, unsigned Length, UIdent Kind,
                                  UIdent AccessLevel,
                                  UIdent SetterAccessLevel,
@@ -2107,10 +2099,6 @@ public:
     if (RespReceiver)
       RespReceiver(createResponse());
   }
-
-  virtual bool forceLibSyntaxBasedProcessing() override {
-    return Opts.ForceLibSyntaxBasedProcessing;
-  }
 };
 
 } // end anonymous namespace

tools/SourceKit/tools/swift-lang/SwiftLang.swift

@@ -41,6 +41,8 @@ public class SwiftLang {
     Request.addParameter(.key_Name, value: name)
     Request.addParameter(.key_SyntaxTreeTransferMode,
                          value: .kind_SyntaxTreeFull)
+    Request.addParameter(.key_EnableSyntaxMap, value: 0)
+    Request.addParameter(.key_EnableStructure, value: 0)
     Request.addParameter(.key_SyntacticOnly, value: 1)
 
     // FIXME: SourceKitd error handling.

tools/swift-ide-test/swift-ide-test.cpp

@@ -39,7 +39,6 @@
 #include "swift/IDE/Utils.h"
 #include "swift/Index/Index.h"
 #include "swift/Sema/IDETypeChecking.h"
-#include "swift/Syntax/SyntaxClassifier.h"
 #include "swift/Markup/Markup.h"
 #include "swift/Config.h"
 #include "clang/APINotes/APINotesReader.h"
@@ -427,12 +426,6 @@ Playground("playground",
            llvm::cl::cat(Category),
            llvm::cl::init(false));
 
-static llvm::cl::opt<bool>
-SyntaxTreeColoring("force-libsyntax-based-processing",
-                   llvm::cl::desc("Perform syntax coloring using libSyntax"),
-                   llvm::cl::cat(Category),
-                   llvm::cl::init(false));
-
 // AST printing options.
 
 static llvm::cl::opt<bool>
@@ -957,125 +950,6 @@ public:
   }
 };
 
-class ColoredSyntaxTreePrinter : public SyntaxVisitor {
-  llvm::raw_ostream &OS;
-
-  std::map<SyntaxNodeId, SyntaxClassification> TokenClassifications;
-
-  /// The name of the tag that is currently open
-  StringRef CurrentTag;
-
-  // Don't print the next newline in trivia if this flag is set.
-  // Used to skip the newline after a CHECK line
-  bool SkipNextNewline = false;
-
-public:
-  ColoredSyntaxTreePrinter(
-      llvm::raw_ostream &OS,
-      std::map<SyntaxNodeId, SyntaxClassification> TokenClassifications)
-      : OS(OS), TokenClassifications(TokenClassifications) {}
-
-private:
-  void print(Trivia Trivia) {
-    for (auto TriviaPiece : Trivia) {
-      print(TriviaPiece);
-    }
-  }
-
-  /// Closes the current tag if it is different from the previous one and opens
-  /// a tag with the specified ID.
-  void recordCurrentTag(StringRef Id) {
-    if (CurrentTag != Id) {
-      if (!CurrentTag.empty()) {
-        OS << "</" << CurrentTag << ">";
-      }
-      if (!Id.empty()) {
-        OS << "<" << Id << ">";
-      }
-    }
-    CurrentTag = Id;
-  }
-
-  void print(TriviaPiece TriviaPiece) {
-    StringRef Id;
-    switch (TriviaPiece.getKind()) {
-    case TriviaKind::Space:
-    case TriviaKind::Tab:
-    case TriviaKind::VerticalTab:
-    case TriviaKind::Formfeed:
-      Id = "";
-      break;
-    case TriviaKind::Newline:
-    case TriviaKind::CarriageReturn:
-    case TriviaKind::CarriageReturnLineFeed:
-      if (SkipNextNewline) {
-        SkipNextNewline = false;
-        return;
-      }
-      Id = "";
-      break;
-    case swift::syntax::TriviaKind::Backtick:
-      Id = "";
-      break;
-    case swift::syntax::TriviaKind::LineComment:
-      // Don't print CHECK lines
-      if (TriviaPiece.getText().contains("// CHECK")) {
-        SkipNextNewline = true;
-        return;
-      }
-      Id = "comment-line";
-      break;
-    case TriviaKind::BlockComment: Id = "comment-block"; break;
-    case TriviaKind::DocLineComment: Id = "doc-comment-line"; break;
-    case TriviaKind::DocBlockComment: Id = "doc-comment-block"; break;
-    case TriviaKind::GarbageText: Id = ""; break;
-    }
-    recordCurrentTag(Id);
-    TriviaPiece.print(OS);
-  }
-
-  StringRef
-  getTagForSyntaxClassification(SyntaxClassification Classification) const {
-    switch (Classification) {
-    case SyntaxClassification::None: return "";
-    case SyntaxClassification::Keyword: return "kw";
-    case SyntaxClassification::Identifier: return "";
-    case SyntaxClassification::DollarIdentifier: return "dollar";
-    case SyntaxClassification::IntegerLiteral: return "int";
-    case SyntaxClassification::FloatingLiteral: return "float";
-    case SyntaxClassification::StringLiteral: return "str";
-    case SyntaxClassification::StringInterpolationAnchor: return "anchor";
-    case SyntaxClassification::TypeIdentifier: return "type";
-    case SyntaxClassification::BuildConfigKeyword: return "#kw";
-    case SyntaxClassification::BuildConfigId: return "#id";
-    case SyntaxClassification::PoundDirectiveKeyword: return "#kw";
-    case SyntaxClassification::Attribute: return "attr-builtin";
-    case SyntaxClassification::EditorPlaceholder: return "placeholder";
-    case SyntaxClassification::ObjectLiteral: return "object-literal";
-    }
-  }
-
-public:
-  virtual void visit(TokenSyntax Token) override {
-    if (Token.isMissing())
-      return;
-
-    print(Token.getLeadingTrivia());
-
-    SyntaxClassification Classification = TokenClassifications[Token.getId()];
-    recordCurrentTag(getTagForSyntaxClassification(Classification));
-    OS << Token.getText();
-
-    print(Token.getTrailingTrivia());
-  }
-
-  void print(Syntax Node) {
-    Node.accept(*this);
-    // Emit the last closing tag
-    recordCurrentTag("");
-  }
-};
-
 } // end anonymous namespace
 
 static int doSyntaxColoring(const CompilerInvocation &InitInvok,
@@ -1110,18 +984,12 @@ static int doSyntaxColoring(const CompilerInvocation &InitInvok,
       break;
   }
   assert(SF && "no source file?");
-  if (options::SyntaxTreeColoring) {
-    SyntaxClassifier Classifier;
-    auto Classification = Classifier.classify(SF->getSyntaxRoot());
-    ColoredSyntaxTreePrinter Printer(llvm::outs(), Classification);
-    Printer.print(SF->getSyntaxRoot());
-  } else {
-    ide::SyntaxModelContext ColorContext(*SF);
-    PrintSyntaxColorWalker ColorWalker(CI.getSourceMgr(), BufID, llvm::outs(),
-                                       TerminalOutput);
-    ColorContext.walk(ColorWalker);
-    ColorWalker.finished();
-  }
+
+  ide::SyntaxModelContext ColorContext(*SF);
+  PrintSyntaxColorWalker ColorWalker(CI.getSourceMgr(), BufID, llvm::outs(),
+                                     TerminalOutput);
+  ColorContext.walk(ColorWalker);
+  ColorWalker.finished();
 
   return 0;
 }

unittests/SourceKit/SwiftLang/CursorInfoTest.cpp

@@ -36,6 +36,8 @@ class NullEditorConsumer : public EditorConsumer {
     llvm_unreachable("unexpected error");
   }
 
+  bool syntaxMapEnabled() override { return true; }
+
   bool handleSyntaxMap(unsigned Offset, unsigned Length, UIdent Kind) override {
     return false;
   }
@@ -45,6 +47,8 @@ class NullEditorConsumer : public EditorConsumer {
     return false;
   }
   
+  bool documentStructureEnabled() override { return false; }
+
   bool beginDocumentSubStructure(unsigned Offset, unsigned Length,
                                  UIdent Kind, UIdent AccessLevel,
                                  UIdent SetterAccessLevel,
@@ -100,8 +104,6 @@ class NullEditorConsumer : public EditorConsumer {
       std::vector<SourceFileRange> ReuseRegions) override {
     return false;
   }
-
-  bool forceLibSyntaxBasedProcessing() override { return false; }
 public:
   bool needsSema = false;
 };

unittests/SourceKit/SwiftLang/EditingTest.cpp

@@ -41,6 +41,8 @@ private:
     llvm_unreachable("unexpected error");
   }
 
+  bool syntaxMapEnabled() override { return true; }
+
   bool handleSyntaxMap(unsigned Offset, unsigned Length, UIdent Kind) override {
     return false;
   }
@@ -50,6 +52,8 @@ private:
     return false;
   }
   
+  bool documentStructureEnabled() override { return false; }
+
   bool beginDocumentSubStructure(unsigned Offset, unsigned Length,
                                  UIdent Kind, UIdent AccessLevel,
                                  UIdent SetterAccessLevel,
@@ -109,8 +113,6 @@ private:
       std::vector<SourceFileRange> ReuseRegions) override {
     return false;
   }
-
-  bool forceLibSyntaxBasedProcessing() override { return false; }
 };
 
 struct DocUpdateMutexState {

utils/gyb_sourcekit_support/UIDs.py

@@ -49,7 +49,6 @@ UID_KEYS = [
     KEY('SerializedSyntaxTree', 'key.serialized_syntax_tree'),
     KEY('SyntaxReuseRegions', 'key.syntaxreuseregions'),
     KEY('SourceText', 'key.sourcetext'),
-    KEY('ForceLibSyntaxBasedProcessing', 'key.forcelibsyntaxbasedprocessing'),
     KEY('EnableSyntaxReuseRegions', 'key.enablesyntaxreuseregions'),
     KEY('EnableSyntaxMap', 'key.enablesyntaxmap'),
     KEY('SyntaxTreeTransferMode', 'key.syntaxtreetransfermode'),