Code completion: add a lot of infrastructure code

* Added a mode in swift-ide-test to test code completion.  Unlike c-index-test,
  the code completion token in tests is a real token -- we don't need to
  count lines and columns anymore.

* Added support in lexer to produce a code completion token.

* Added a parser interface to code completion.  It is passed down from the
  libFrontend to the parser, but its functions are not called yet.

* Added a sketch of the interface of code completion consumer and code
  completion results.

Note: all this is not doing anything useful yet.


Swift SVN r6128

lib/IDE/CodeCompletion.cpp

@@ -0,0 +1,98 @@
+#include "swift/IDE/CodeCompletion.h"
+#include "swift/Basic/LLVM.h"
+#include "swift/Parse/CodeCompletionCallbacks.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/raw_ostream.h"
+#include <string>
+#include <algorithm>
+
+using namespace swift;
+using namespace code_completion;
+
+std::string swift::code_completion::removeCodeCompletionTokens(
+    StringRef Input, StringRef TokenName, unsigned *CompletionOffset) {
+  assert(TokenName.size() >= 1);
+
+  *CompletionOffset = ~0U;
+
+  std::string CleanFile;
+  CleanFile.reserve(Input.size());
+  const std::string Token = std::string("#^") + TokenName.str() + "^#";
+
+  for (const char *Ptr = Input.begin(), *End = Input.end();
+       Ptr != End; ++Ptr) {
+    const char C = *Ptr;
+    if (C == '#' && Ptr <= End - Token.size() &&
+        StringRef(Ptr, Token.size()) == Token) {
+      Ptr += Token.size() - 1;
+      *CompletionOffset = CleanFile.size() + 1;
+      continue;
+    }
+    if (C == '#' && Ptr <= End - 2 && Ptr[1] == '^') {
+      do {
+        Ptr++;
+      } while(*Ptr != '#');
+      continue;
+    }
+    CleanFile += C;
+  }
+  return CleanFile;
+}
+
+StringRef CodeCompletionContext::CopyString(StringRef String) {
+  char *Mem = (char *) Allocator.Allocate(String.size(), 1);
+  std::copy(String.begin(), String.end(), Mem);
+  return StringRef(Mem, String.size());
+}
+
+namespace {
+class CodeCompletionCallbacksImpl : public CodeCompletionCallbacks {
+  CodeCompletionContext &CompletionContext;
+  CodeCompletionConsumer &Consumer;
+
+public:
+  CodeCompletionCallbacksImpl(Parser &P,
+                              CodeCompletionContext &CompletionContext,
+                              CodeCompletionConsumer &Consumer)
+      : CodeCompletionCallbacks(P), CompletionContext(CompletionContext),
+        Consumer(Consumer) {
+  }
+
+  void completeExpr() override;
+};
+} // end unnamed namespace
+
+void CodeCompletionCallbacksImpl::completeExpr() {
+  (void) CompletionContext; // Silence "unused member variable" warning.
+  Consumer.handleResults(ArrayRef<CodeCompletionResult>());
+}
+
+void PrintingCodeCompletionConsumer::handleResults(
+    ArrayRef<CodeCompletionResult> Results) {
+  OS << "hello completions\n";
+}
+
+namespace {
+class CodeCompletionCallbacksFactoryImpl
+    : public CodeCompletionCallbacksFactory {
+  CodeCompletionContext &CompletionContext;
+  CodeCompletionConsumer &Consumer;
+
+public:
+  CodeCompletionCallbacksFactoryImpl(CodeCompletionContext &CompletionContext,
+                                     CodeCompletionConsumer &Consumer)
+      : CompletionContext(CompletionContext), Consumer(Consumer) {}
+
+  CodeCompletionCallbacks *createCodeCompletionCallbacks(Parser &P) override {
+    return new CodeCompletionCallbacksImpl(P, CompletionContext, Consumer);
+  }
+};
+} // end unnamed namespace
+
+CodeCompletionCallbacksFactory *
+swift::code_completion::makeCodeCompletionCallbacksFactory(
+    CodeCompletionContext &CompletionContext,
+    CodeCompletionConsumer &Consumer) {
+  return new CodeCompletionCallbacksFactoryImpl(CompletionContext, Consumer);
+}
+