Code completion: add a lot of infrastructure code

* Added a mode in swift-ide-test to test code completion.  Unlike c-index-test,
  the code completion token in tests is a real token -- we don't need to
  count lines and columns anymore.

* Added support in lexer to produce a code completion token.

* Added a parser interface to code completion.  It is passed down from the
  libFrontend to the parser, but its functions are not called yet.

* Added a sketch of the interface of code completion consumer and code
  completion results.

Note: all this is not doing anything useful yet.


Swift SVN r6128

unittests/IDE/CodeCompletionToken.cpp

@@ -0,0 +1,65 @@
+#include "swift/IDE/CodeCompletion.h"
+#include "gtest/gtest.h"
+
+using namespace swift;
+using namespace code_completion;
+
+TEST(CodeCompletionToken, FindInEmptyFile) {
+  std::string Source = "";
+  unsigned Offset;
+  std::string Clean = removeCodeCompletionTokens(Source, "A", &Offset);
+  EXPECT_EQ(~0U, Offset);
+  EXPECT_EQ("", Clean);
+}
+
+TEST(CodeCompletionToken, FindNonExistent) {
+  std::string Source = "func zzz() {}";
+  unsigned Offset;
+  std::string Clean = removeCodeCompletionTokens(Source, "A", &Offset);
+  EXPECT_EQ(~0U, Offset);
+  EXPECT_EQ(Source, Clean);
+}
+
+TEST(CodeCompletionToken, RemovesOtherTokens) {
+  std::string Source = "func zzz() {#^B^#}";
+  unsigned Offset;
+  std::string Clean = removeCodeCompletionTokens(Source, "A", &Offset);
+  EXPECT_EQ(~0U, Offset);
+  EXPECT_EQ("func zzz() {}", Clean);
+}
+
+TEST(CodeCompletionToken, FindBegin) {
+  std::string Source = "#^A^# func";
+  unsigned Offset;
+  std::string Clean = removeCodeCompletionTokens(Source, "A", &Offset);
+  EXPECT_EQ(1U, Offset);
+  EXPECT_EQ(" func", Clean);
+}
+
+TEST(CodeCompletionToken, FindEnd) {
+  std::string Source = "func #^A^#";
+  unsigned Offset;
+  std::string Clean = removeCodeCompletionTokens(Source, "A", &Offset);
+  EXPECT_EQ(6U, Offset);
+  EXPECT_EQ("func ", Clean);
+}
+
+TEST(CodeCompletionToken, FindSingleLine) {
+  std::string Source = "func zzz() {#^A^#}";
+  unsigned Offset;
+  std::string Clean = removeCodeCompletionTokens(Source, "A", &Offset);
+  EXPECT_EQ(13U, Offset);
+  EXPECT_EQ("func zzz() {}", Clean);
+}
+
+TEST(CodeCompletionToken, FindMultiline) {
+  std::string Source =
+      "func zzz() {\n"
+      "  1 + #^A^#\r\n"
+      "}\n";
+  unsigned Offset;
+  std::string Clean = removeCodeCompletionTokens(Source, "A", &Offset);
+  EXPECT_EQ(20U, Offset);
+  EXPECT_EQ("func zzz() {\n  1 + \r\n}\n", Clean);
+}
+