Merge pull request #22835 from nkcsgexi/custom-buffer

sourcekitd: define a new custom buffer for expression type response.

lib/IDE/IDETypeChecking.cpp

@@ -638,11 +638,13 @@ public:
     {
       llvm::raw_svector_ostream OS(Buffer);
       E->getType()->getRValueType()->reconstituteSugar(true)->print(OS);
+      // Ensure the end user can directly use the char*
+      OS << '\0';
     }
 
     // Add the type information to the result list.
     Results.push_back({Offset, Length, getTypeOffsets(Buffer.str()),
-      static_cast<uint32_t>(Buffer.size())});
+      static_cast<uint32_t>(Buffer.size()) - 1});
 
     // Keep track of that we have a type reported for this range.
     AllPrintedTypes[Offset].insert(Length);

test/SourceKit/ExpressionType/basic.swift

@@ -1,4 +1,34 @@
-func foo() { return 1 }
+func foo() -> Int  { return 1 }
+
+func bar(f: Float) -> Float { return bar(f: 1) }
+
+protocol P {}
+
+func fooP(_ p: P) { fooP(p) }
+
+class C {}
+
+func ArrayC(_ a: [C]) {
+	_ = a.count
+	_ = a.description.count.advanced(by: 1).description
+}
+
+struct S {
+  let val = 4
+}
+func DictS(_ a: [Int: S]) {
+  _ = a[2]?.val.advanced(by: 1).byteSwapped
+}
 
 // RUN: %sourcekitd-test -req=collect-type %s -- %s | %FileCheck %s
-// CHECK: (20, 21): Int
+// CHECK: (183, 202): Int
+// CHECK: (183, 196): String
+// CHECK: (183, 184): [C]
+// CHECK: (203, 211): (Int) -> (Int) -> Int
+// CHECK: (211, 218): (by: Int)
+// CHECK: (216, 217): Int
+// CHECK: (257, 258): Int
+// CHECK: (291, 332): ()
+// CHECK: (291, 292): Int?
+// CHECK: (295, 332): Int?
+// CHECK: (295, 320): Int

tools/SourceKit/include/SourceKit/Core/LangSupport.h

@@ -128,7 +128,6 @@ struct ExpressionType {
   unsigned ExprOffset;
   unsigned ExprLength;
   unsigned TypeOffset;
-  unsigned TypeLength;
 };
 
 struct ExpressionTypesInFile {

tools/SourceKit/lib/SwiftLang/SwiftSourceDocInfo.cpp

@@ -2057,8 +2057,7 @@ void SwiftLangSupport::collectExpressionTypes(StringRef FileName,
       llvm::raw_svector_ostream OS(TypeBuffer);
       ExpressionTypesInFile Result;
       for (auto Item: collectExpressionType(*SF, Scratch, OS)) {
-        Result.Results.push_back({Item.offset, Item.length, Item.typeOffset,
-          Item.typeLength});
+        Result.Results.push_back({Item.offset, Item.length, Item.typeOffset});
       }
       Result.TypeBuffer = OS.str();
       Receiver(Result);

tools/SourceKit/tools/sourcekitd-test/sourcekitd-test.cpp

@@ -1537,20 +1537,18 @@ static void printRangeInfo(sourcekitd_variant_t Info, StringRef FilenameIn,
 }
 
 static void printExpressionType(sourcekitd_variant_t Info, llvm::raw_ostream &OS) {
-  auto *TypeBuffer = sourcekitd_variant_dictionary_get_string(Info, KeyTypeBuffer);
-  sourcekitd_variant_t ExprList = sourcekitd_variant_dictionary_get_value(Info,
-    KeyExpressionTypeList);
-  unsigned Count = sourcekitd_variant_array_get_count(ExprList);
-  for (unsigned i = 0; i != Count; ++i) {
-    sourcekitd_variant_t Item = sourcekitd_variant_array_get_value(ExprList, i);
-    unsigned Offset = sourcekitd_variant_dictionary_get_int64(Item, KeyExpressionOffset);
-    unsigned Length = sourcekitd_variant_dictionary_get_int64(Item, KeyExpressionLength);
-    StringRef PrintedType(TypeBuffer + sourcekitd_variant_dictionary_get_int64(Item,
-      KeyTypeOffset), sourcekitd_variant_dictionary_get_int64(Item, KeyTypeLength));
-    OS << "(" << Offset << ", " << Offset + Length << "): " << PrintedType << "\n";
-  }
+  auto TypeBuffer = sourcekitd_variant_dictionary_get_value(Info, KeyExpressionTypeList);
+  unsigned Count = sourcekitd_variant_array_get_count(TypeBuffer);
   if (!Count) {
     OS << "cannot find expression types in the file\n";
+    return;
+  }
+  for (unsigned i = 0; i != Count; ++i) {
+    sourcekitd_variant_t Item = sourcekitd_variant_array_get_value(TypeBuffer, i);
+    unsigned Offset = sourcekitd_variant_dictionary_get_int64(Item, KeyExpressionOffset);
+    unsigned Length = sourcekitd_variant_dictionary_get_int64(Item, KeyExpressionLength);
+    OS << "(" << Offset << ", " << Offset + Length << "): " <<
+      sourcekitd_variant_dictionary_get_string(Item, KeyExpressionType) << "\n";
   }
 }
 

tools/SourceKit/tools/sourcekitd/include/sourcekitd/CompactArray.h

@@ -22,7 +22,7 @@ class CompactArrayBuilderImpl {
 public:
   std::unique_ptr<llvm::MemoryBuffer> createBuffer() const;
   void appendTo(llvm::SmallVectorImpl<char> &Buf) const;
-
+  unsigned copyInto(char *BufPtr) const;
   size_t sizeInBytes() const;
   bool empty() const;
 

tools/SourceKit/tools/sourcekitd/include/sourcekitd/ExpressionTypeArray.h

@@ -0,0 +1,41 @@
+//===--- ExpressionTypeArray.h - --------------------------------*- C++ -*-===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SOURCEKITD_EXPRESSION_TYPE_ARRAY_H
+#define LLVM_SOURCEKITD_EXPRESSION_TYPE_ARRAY_H
+
+#include "sourcekitd/Internal.h"
+
+namespace SourceKit {
+  struct ExpressionType;
+}
+
+namespace sourcekitd {
+VariantFunctions *getVariantFunctionsForExpressionTypeArray();
+
+class ExpressionTypeArrayBuilder {
+public:
+  ExpressionTypeArrayBuilder(llvm::StringRef PrintedType);
+  ~ExpressionTypeArrayBuilder();
+
+  void add(const SourceKit::ExpressionType &ExpType);
+  std::unique_ptr<llvm::MemoryBuffer> createBuffer();
+  static VariantFunctions Funcs;
+
+private:
+  struct Implementation;
+  Implementation &Impl;
+};
+
+}
+
+#endif

tools/SourceKit/tools/sourcekitd/include/sourcekitd/Internal.h

@@ -59,6 +59,7 @@ enum class CustomBufferKind {
   InheritedTypesArray,
   DocStructureElementArray,
   AttributesArray,
+  ExpressionTypeArray,
   RawData
 };
 

tools/SourceKit/tools/sourcekitd/lib/API/CMakeLists.txt

@@ -8,6 +8,7 @@ set(sourcekitdAPI_sources
   Requests.cpp
   sourcekitdAPI-Common.cpp
   TokenAnnotationsArray.cpp
+  ExpressionTypeArray.cpp
 )
 
 set(sourcekitdAPI_Darwin_sources

tools/SourceKit/tools/sourcekitd/lib/API/CompactArray.cpp

@@ -94,6 +94,12 @@ void CompactArrayBuilderImpl::copyInto(char *BufPtr, size_t Length) const {
   memcpy(BufPtr, StringBuffer.data(), StringBuffer.size());
 }
 
+unsigned CompactArrayBuilderImpl::copyInto(char *BufPtr) const {
+  size_t Length = sizeInBytes();
+  copyInto(BufPtr, Length);
+  return Length;
+}
+
 bool CompactArrayBuilderImpl::empty() const {
   return EntriesBuffer.empty();
 }

tools/SourceKit/tools/sourcekitd/lib/API/ExpressionTypeArray.cpp

@@ -0,0 +1,161 @@
+//===---------- ExpressionTypeArray.cpp -----------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+#include "sourcekitd/ExpressionTypeArray.h"
+#include "sourcekitd/CompactArray.h"
+#include "SourceKit/Core/LLVM.h"
+#include "SourceKit/Core/LangSupport.h"
+#include "SourceKit/Support/UIdent.h"
+#include "DictionaryKeys.h"
+
+#include "llvm/Support/MemoryBuffer.h"
+
+using namespace SourceKit;
+using namespace sourcekitd;
+
+class ExpressionTypeReader {
+  const char *printedType;
+  CompactArrayReader<unsigned, unsigned, unsigned> entryReader;
+
+  static uint64_t getHeaderValue(char *buffer, unsigned index) {
+    uint64_t headerField;
+    memcpy(&headerField, (uint64_t*)buffer + index, sizeof(headerField));
+    return headerField;
+  }
+
+public:
+  ExpressionTypeReader(char *buffer):
+      entryReader(buffer + getHeaderValue(buffer, 0)) {
+    // Read the printed type string buffer here.
+    CompactArrayReader<const char*> reader(buffer + getHeaderValue(buffer, 1));
+    reader.readEntries(0, printedType);
+  }
+
+  uint64_t count() const { return entryReader.getCount(); }
+
+  std::pair<std::pair<unsigned, unsigned>, const char*> getExpression(uint64_t idx) {
+    ExpressionType result;
+    entryReader.readEntries(idx, result.ExprOffset, result.ExprLength,
+                            result.TypeOffset);
+    return {std::make_pair(result.ExprOffset, result.ExprLength),
+      printedType + result.TypeOffset};
+  }
+
+  static bool
+  dictionary_apply(void *buffer, size_t index,
+                   llvm::function_ref<bool(sourcekitd_uid_t,
+                                           sourcekitd_variant_t)> applier) {
+    ExpressionTypeReader reader((char*)buffer);
+    auto result = reader.getExpression(index);
+#define APPLY(K, Ty, Field)                              \
+  do {                                                   \
+    sourcekitd_uid_t key = SKDUIDFromUIdent(K);          \
+    sourcekitd_variant_t var = make##Ty##Variant(Field); \
+    if (!applier(key, var)) return false;                \
+  } while (0)
+    APPLY(KeyExpressionOffset, Int, result.first.first);
+    APPLY(KeyExpressionLength, Int, result.first.second);
+    APPLY(KeyExpressionType, String, result.second);
+    return true;
+  }
+};
+
+struct ExpressionTypeArrayBuilder::Implementation {
+  StringRef printedType;
+  SmallVector<char, 256> buffer;
+  CompactArrayBuilder<unsigned, unsigned, unsigned> builder;
+  CompactArrayBuilder<StringRef> strBuilder;
+
+  Implementation(StringRef PrintedType) { strBuilder.addEntry(PrintedType); }
+  static sourcekitd_variant_type_t get_type(sourcekitd_variant_t var) {
+    return SOURCEKITD_VARIANT_TYPE_ARRAY;
+  }
+
+  static sourcekitd_variant_t array_get_value(sourcekitd_variant_t array,
+                                              size_t index) {
+    return {{(uintptr_t)&CompactVariantFuncs<ExpressionTypeReader>::Funcs,
+      (uintptr_t)array.data[1], index}};
+  }
+
+  // data[0] = ExpressionTypeArrayBuilder::funcs
+  // data[1] = custum buffer
+  static size_t array_get_count(sourcekitd_variant_t array) {
+    ExpressionTypeReader reader((char*)array.data[1]);
+    return reader.count();
+  }
+
+  std::unique_ptr<llvm::MemoryBuffer> createBuffer() {
+    size_t headerSize = sizeof(uint64_t) * 2;
+    auto result = llvm::WritableMemoryBuffer::getNewUninitMemBuffer(
+      headerSize + builder.sizeInBytes() + strBuilder.sizeInBytes());
+    char *start = result->getBufferStart();
+    char *headerPtr = start;
+    char *ptr = start + headerSize;
+    auto addBuilder = [&](CompactArrayBuilderImpl& buffer) {
+      uint64_t offset = ptr - start;
+      memcpy(headerPtr, &offset, sizeof(offset));
+      headerPtr += sizeof(offset);
+      ptr += buffer.copyInto(ptr);
+    };
+
+    addBuilder(builder);
+    addBuilder(strBuilder);
+    assert(ptr == result->getBufferEnd());
+    return std::move(result);
+  }
+};
+
+ExpressionTypeArrayBuilder::ExpressionTypeArrayBuilder(StringRef printedType)
+  : Impl(*new Implementation(printedType)) {}
+
+ExpressionTypeArrayBuilder::~ExpressionTypeArrayBuilder() {
+  delete &Impl;
+}
+
+void ExpressionTypeArrayBuilder::add(const ExpressionType &expType) {
+  Impl.builder.addEntry(expType.ExprOffset, expType.ExprLength,
+                        expType.TypeOffset/*Printed type is null ended*/);
+}
+
+std::unique_ptr<llvm::MemoryBuffer>
+ExpressionTypeArrayBuilder::createBuffer() {
+  return Impl.createBuffer();
+}
+
+VariantFunctions ExpressionTypeArrayBuilder::Funcs = {
+  Implementation::get_type,
+  nullptr /*AnnotArray_array_apply*/,
+  nullptr /*AnnotArray_array_get_bool*/,
+  Implementation::array_get_count,
+  nullptr /*AnnotArray_array_get_int64*/,
+  nullptr /*AnnotArray_array_get_string*/,
+  nullptr /*AnnotArray_array_get_uid*/,
+  Implementation::array_get_value,
+  nullptr /*AnnotArray_bool_get_value*/,
+  nullptr /*AnnotArray_dictionary_apply*/,
+  nullptr /*AnnotArray_dictionary_get_bool*/,
+  nullptr /*AnnotArray_dictionary_get_int64*/,
+  nullptr /*AnnotArray_dictionary_get_string*/,
+  nullptr /*AnnotArray_dictionary_get_value*/,
+  nullptr /*AnnotArray_dictionary_get_uid*/,
+  nullptr /*AnnotArray_string_get_length*/,
+  nullptr /*AnnotArray_string_get_ptr*/,
+  nullptr /*AnnotArray_int64_get_value*/,
+  nullptr /*AnnotArray_uid_get_value*/,
+  nullptr /*Annot_data_get_size*/,
+  nullptr /*Annot_data_get_ptr*/,
+};
+
+VariantFunctions *
+sourcekitd::getVariantFunctionsForExpressionTypeArray() {
+ return &ExpressionTypeArrayBuilder::Funcs;
+}

tools/SourceKit/tools/sourcekitd/lib/API/Requests.cpp

@@ -15,6 +15,7 @@
 #include "sourcekitd/DocStructureArray.h"
 #include "sourcekitd/DocSupportAnnotationArray.h"
 #include "sourcekitd/TokenAnnotationsArray.h"
+#include "sourcekitd/ExpressionTypeArray.h"
 
 #include "SourceKit/Core/Context.h"
 #include "SourceKit/Core/LangSupport.h"
@@ -1742,15 +1743,12 @@ static void reportExpressionTypeInfo(const ExpressionTypesInFile &Info,
                                      ResponseReceiver Rec) {
   ResponseBuilder Builder;
   auto Dict = Builder.getDictionary();
-  Dict.set(KeyTypeBuffer, Info.TypeBuffer);
-  ResponseBuilder::Array Arr = Dict.setArray(KeyExpressionTypeList);
-  for (auto Result: Info.Results) {
-    auto Elem = Arr.appendDictionary();
-    Elem.set(KeyExpressionOffset, Result.ExprOffset);
-    Elem.set(KeyExpressionLength, Result.ExprLength);
-    Elem.set(KeyTypeOffset, Result.TypeOffset);
-    Elem.set(KeyTypeLength, Result.TypeLength);
+  ExpressionTypeArrayBuilder ArrBuilder(Info.TypeBuffer);
+  for (auto &R: Info.Results) {
+    ArrBuilder.add(R);
   }
+  Dict.setCustomBuffer(KeyExpressionTypeList, CustomBufferKind::ExpressionTypeArray,
+                       ArrBuilder.createBuffer());
   Rec(Builder.createResponse());
 }
 

tools/SourceKit/tools/sourcekitd/lib/API/sourcekitdAPI-InProc.cpp

@@ -17,6 +17,7 @@
 #include "sourcekitd/DocSupportAnnotationArray.h"
 #include "sourcekitd/RawData.h"
 #include "sourcekitd/TokenAnnotationsArray.h"
+#include "sourcekitd/ExpressionTypeArray.h"
 #include "sourcekitd/Logging.h"
 #include "SourceKit/Core/LLVM.h"
 #include "SourceKit/Support/UIdent.h"
@@ -256,6 +257,7 @@ public:
       case CustomBufferKind::InheritedTypesArray:
       case CustomBufferKind::DocStructureElementArray:
       case CustomBufferKind::AttributesArray:
+      case CustomBufferKind::ExpressionTypeArray:
         return SOURCEKITD_VARIANT_TYPE_ARRAY;
       case CustomBufferKind::RawData:
         return SOURCEKITD_VARIANT_TYPE_DATA;
@@ -982,6 +984,9 @@ static sourcekitd_variant_t variantFromSKDObject(SKDObjectRef Object) {
       case CustomBufferKind::AttributesArray:
         return {{ (uintptr_t)getVariantFunctionsForAttributesArray(),
           (uintptr_t)DataObject->getDataPtr(), 0 }};
+      case CustomBufferKind::ExpressionTypeArray:
+        return {{ (uintptr_t)getVariantFunctionsForExpressionTypeArray(),
+          (uintptr_t)DataObject->getDataPtr(), 0 }};
       case CustomBufferKind::RawData:
         return {{ (uintptr_t)getVariantFunctionsForRawData(),
                   (uintptr_t)DataObject->getDataPtr(),

tools/SourceKit/tools/sourcekitd/lib/API/sourcekitdAPI-XPC.cpp

@@ -15,6 +15,7 @@
 #include "sourcekitd/DocStructureArray.h"
 #include "sourcekitd/DocSupportAnnotationArray.h"
 #include "sourcekitd/TokenAnnotationsArray.h"
+#include "sourcekitd/ExpressionTypeArray.h"
 #include "sourcekitd/RawData.h"
 #include "sourcekitd/RequestResponsePrinterBase.h"
 #include "SourceKit/Support/UIdent.h"
@@ -621,6 +622,7 @@ static sourcekitd_variant_type_t XPCVar_get_type(sourcekitd_variant_t var) {
     case CustomBufferKind::InheritedTypesArray:
     case CustomBufferKind::DocStructureElementArray:
     case CustomBufferKind::AttributesArray:
+    case CustomBufferKind::ExpressionTypeArray:
       return SOURCEKITD_VARIANT_TYPE_ARRAY;
     case CustomBufferKind::RawData:
       return SOURCEKITD_VARIANT_TYPE_DATA;
@@ -781,6 +783,9 @@ static sourcekitd_variant_t variantFromXPCObject(xpc_object_t obj) {
     case CustomBufferKind::AttributesArray:
       return {{ (uintptr_t)getVariantFunctionsForAttributesArray(),
                 (uintptr_t)CUSTOM_BUF_START(obj), 0 }};
+    case CustomBufferKind::ExpressionTypeArray:
+      return {{ (uintptr_t)getVariantFunctionsForExpressionTypeArray(),
+                (uintptr_t)CUSTOM_BUF_START(obj), 0 }};
     case sourcekitd::CustomBufferKind::RawData:
       return {{ (uintptr_t)getVariantFunctionsForRawData(),
                 (uintptr_t)CUSTOM_BUF_START(obj),

utils/gyb_sourcekit_support/UIDs.py

@@ -168,8 +168,7 @@ UID_KEYS = [
     KEY('ExpressionTypeList', 'key.expression_type_list'),
     KEY('ExpressionOffset', 'key.expression_offset'),
     KEY('ExpressionLength', 'key.expression_length'),
-    KEY('TypeOffset', 'key.type_offset'),
-    KEY('TypeLength', 'key.type_length'),
+    KEY('ExpressionType', 'key.expression_type'),
 ]