[SourceKit] Add the raw doc comment to the cursor info response

SourceKit-LSP currently parses the XML comment to generate Markdown again but round-tripping a (probably markdown) doc comment to XML to Markdown is lossy in many cases and unnecessary work. Include the comment as it is spelled in source in the cursor info response so that sourcekit-lsp can display it.

Part of rdar://120685874

include/swift/IDE/CommentConversion.h

@@ -32,6 +32,12 @@ bool getDocumentationCommentAsXML(
   const Decl *D, raw_ostream &OS,
   TypeOrExtensionDecl SynthesizedTarget = TypeOrExtensionDecl());
 
+/// If the declaration has a documentation comment, prints the comment to \p OS
+/// in the form it's written in source.
+///
+/// \returns true if the declaration has a documentation comment.
+bool getRawDocumentationComment(const Decl *D, raw_ostream &OS);
+
 /// If the declaration has a documentation comment and a localization key,
 /// print it into the given output stream and return true. Else, return false.
 bool getLocalizationKey(const Decl *D, raw_ostream &OS);

lib/IDE/CommentConversion.cpp

@@ -24,6 +24,7 @@
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/raw_ostream.h"
 #include "clang/AST/ASTContext.h"
+#include "clang/AST/Comment.h"
 #include "clang/AST/Decl.h"
 #include "clang/Index/CommentToXML.h"
 
@@ -496,6 +497,38 @@ bool ide::getDocumentationCommentAsXML(const Decl *D, raw_ostream &OS,
   return true;
 }
 
+bool ide::getRawDocumentationComment(const Decl *D, raw_ostream &OS) {
+  ClangNode MaybeClangNode = D->getClangNode();
+  if (MaybeClangNode) {
+    const clang::Decl *CD = MaybeClangNode.getAsDecl();
+    if (!CD) {
+      return false;
+    }
+    const clang::ASTContext &ClangContext = CD->getASTContext();
+    const clang::comments::FullComment *FC =
+      ClangContext.getCommentForDecl(CD, /*PP=*/nullptr);
+    if (!FC) {
+      return false;
+    }
+    const clang::RawComment *rawComment = ClangContext.getRawCommentForAnyRedecl(FC->getDecl());
+    if (!rawComment) {
+      return false;
+    }
+    OS << rawComment->getFormattedText(ClangContext.getSourceManager(),
+                                       ClangContext.getDiagnostics());
+    return true;
+  }
+
+  const Decl *docD = getDocCommentProvidingDecl(D);
+  if (!docD) {
+    return false;
+  }
+  RawComment rawComment = docD->getRawComment();
+  OS << swift::markup::MarkupContext().getLineList(rawComment).str();
+  OS.flush();
+  return true;
+}
+
 bool ide::getLocalizationKey(const Decl *D, raw_ostream &OS) {
   swift::markup::MarkupContext MC;
   auto DC = getCascadingDocComment(MC, D);

lib/Markup/LineList.cpp

@@ -24,18 +24,19 @@ std::string LineList::str() const {
   if (Lines.empty())
     return "";
 
-  auto FirstLine = Lines.begin();
-  while (FirstLine != Lines.end() && FirstLine->Text.empty())
-    ++FirstLine;
+  Line *FirstNonEmptyLine = Lines.begin();
+  while (FirstNonEmptyLine != Lines.end() && FirstNonEmptyLine->Text.empty())
+    ++FirstNonEmptyLine;
 
-  if (FirstLine == Lines.end())
+  if (FirstNonEmptyLine == Lines.end())
     return "";
 
-  auto InitialIndentation = measureIndentation(FirstLine->Text);
+  auto InitialIndentation = measureIndentation(FirstNonEmptyLine->Text);
 
-  for (auto Line = FirstLine; Line != Lines.end(); ++Line) {
+  Stream << FirstNonEmptyLine->Text.drop_front(InitialIndentation);
+  for (auto Line = FirstNonEmptyLine + 1; Line != Lines.end(); ++Line) {
     auto Drop = std::min(InitialIndentation, Line->FirstNonspaceOffset);
-    Stream << Line->Text.drop_front(Drop) << "\n";
+    Stream << '\n' << Line->Text.drop_front(Drop);
   }
 
   Stream.flush();

test/SourceKit/CursorInfo/cursor_doc_comment.swift

@@ -0,0 +1,11 @@
+/// Test
+/// - Returns: An integer
+func test() -> Int {}
+// RUN: %sourcekitd-test -req=cursor -pos=%(line - 1):6 %s -- %s | %FileCheck %s
+
+// CHECK-LABEL: DOC COMMENT
+// CHECK: Test
+// CHECK: - Returns: An integer
+
+// CHECK-LABEL: DOC COMMENT XML
+// CHECK: <Function file="{{.*}}" line="3" column="6"><Name>test()</Name><USR>s:18cursor_doc_comment4testSiyF</USR><Declaration>func test() -&gt; Int</Declaration><CommentParts><Abstract><Para>Test</Para></Abstract><ResultDiscussion><Para>An integer</Para></ResultDiscussion></CommentParts></Function>

test/SourceKit/CursorInfo/cursor_doc_comment_from_clang.swift

@@ -0,0 +1,28 @@
+// RUN: %empty-directory(%t)
+// RUN: split-file --leading-lines %s %t
+
+//--- header.h
+
+/// This comment contains `markup`.
+///
+/// - And a list
+void testCDecl();
+
+//--- module.modulemap
+
+module MyClangModule { header "header.h" }
+
+//--- test.swift
+
+import MyClangModule
+
+func test() {
+  // RUN: %sourcekitd-test -req=cursor -pos=%(line + 1):3 %s -- %s -I %t | %FileCheck %s
+  testCDecl()
+}
+
+// CHECK-LABEL: DOC COMMENT
+// CHECK: This comment contains `markup`.
+// CHECK: - And a list
+// CHECK-LABEL: DOC COMMENT XML
+// CHECK: <Function file="{{.*}}" line="9" column="6"><Name>testCDecl</Name><USR>c:@F@testCDecl</USR><Declaration>func testCDecl()</Declaration><Abstract><Para> This comment contains `markup`.</Para></Abstract><Discussion><Para> - And a list</Para></Discussion></Function>

test/SourceKit/CursorInfo/cursor_info.swift

@@ -288,6 +288,9 @@ let strInterpolation = "This is a \(stringStr + "ing") interpolation"
 // CHECK4-NEXT: Foo{{$}}
 // CHECK4-NEXT: <Declaration>var fooIntVar: <Type usr="s:s5Int32V">Int32</Type></Declaration>
 // CHECK4-NEXT: <decl.var.global><syntaxtype.keyword>var</syntaxtype.keyword> <decl.name>fooIntVar</decl.name>: <decl.var.type><ref.struct usr="s:s5Int32V">Int32</ref.struct></decl.var.type></decl.var.global>
+// CHECK4-NEXT: DOC COMMENT 
+// CHECK4-NEXT: Aaa. fooIntVar. Bbb. 
+// CHECK4-NEXT: DOC COMMENT XML
 // CHECK4-NEXT: <Variable file="{{[^"]+}}Foo.h" line="{{[0-9]+}}" column="{{[0-9]+}}"><Name>fooIntVar</Name><USR>c:@fooIntVar</USR><Declaration>var fooIntVar: Int32</Declaration><Abstract><Para> Aaa. fooIntVar. Bbb.</Para></Abstract></Variable>
 
 // RUN: %sourcekitd-test -req=cursor -pos=8:7 %s -- -F %S/../Inputs/libIDE-mock-sdk -I %t.tmp %s | %FileCheck -check-prefix=CHECK5 %s
@@ -307,6 +310,9 @@ let strInterpolation = "This is a \(stringStr + "ing") interpolation"
 // CHECK6-NEXT: FooSwiftModule
 // CHECK6-NEXT: <Declaration>func fooSwiftFunc() -&gt; <Type usr="s:Si">Int</Type></Declaration>
 // CHECK6-NEXT: <decl.function.free><syntaxtype.keyword>func</syntaxtype.keyword> <decl.name>fooSwiftFunc</decl.name>() -&gt; <decl.function.returntype><ref.struct usr="s:Si">Int</ref.struct></decl.function.returntype></decl.function.free>
+// CHECK6-NEXT: DOC COMMENT 
+// CHECK6-NEXT: This is 'fooSwiftFunc' from 'FooSwiftModule'. 
+// CHECK6-NEXT: DOC COMMENT XML
 // CHECK6-NEXT: {{^}}<Function file="{{.*}}/FooSwiftModule.swift" line="2" column="13"><Name>fooSwiftFunc()</Name><USR>s:14FooSwiftModule03fooB4FuncSiyF</USR><Declaration>func fooSwiftFunc() -&gt; Int</Declaration><CommentParts><Abstract><Para>This is ‘fooSwiftFunc’ from ‘FooSwiftModule’.</Para></Abstract></CommentParts></Function>{{$}}
 
 // RUN: %sourcekitd-test -req=cursor -pos=14:10 %s -- -F %S/../Inputs/libIDE-mock-sdk -I %t.tmp %s | %FileCheck -check-prefix=CHECK7 %s
@@ -319,6 +325,9 @@ let strInterpolation = "This is a \(stringStr + "ing") interpolation"
 // CHECK7-NEXT: cursor_info{{$}}
 // CHECK7-NEXT: <Declaration>struct S1</Declaration>
 // CHECK7-NEXT: <decl.struct><syntaxtype.keyword>struct</syntaxtype.keyword> <decl.name>S1</decl.name></decl.struct>
+// CHECK7-NEXT: DOC COMMENT 
+// CHECK7-NEXT: Aaa. S1. Bbb. 
+// CHECK7-NEXT: DOC COMMENT XML
 // CHECK7-NEXT: <Class file="{{[^"]+}}cursor_info.swift" line="13" column="8"><Name>S1</Name><USR>s:11cursor_info2S1V</USR><Declaration>struct S1</Declaration><CommentParts><Abstract><Para>Aaa.  S1.  Bbb.</Para></Abstract></CommentParts></Class>
 
 // RUN: %sourcekitd-test -req=cursor -pos=19:12 %s -- -F %S/../Inputs/libIDE-mock-sdk -I %t.tmp %s | %FileCheck -check-prefix=CHECK8 %s
@@ -780,6 +789,11 @@ let strInterpolation = "This is a \(stringStr + "ing") interpolation"
 // CHECK87-NEXT: cursor_info{{$}}
 // CHECK87-NEXT: <Declaration>struct HasLocalizationKey</Declaration>
 // CHECK87-NEXT: <decl.struct><syntaxtype.keyword>struct</syntaxtype.keyword> <decl.name>HasLocalizationKey</decl.name></decl.struct>
+// CHECK87-NEXT: DOC COMMENT
+// CHECK87-NEXT: Brief. 
+// CHECK87-EMPTY:  
+// CHECK87-NEXT: - LocalizationKey: ABC 
+// CHECK87-NEXT: DOC COMMENT XML
 // CHECK87-NEXT: <Class file="{{[^"]+}}cursor_info.swift" line="213" column="8"><Name>HasLocalizationKey</Name><USR>s:11cursor_info18HasLocalizationKeyV</USR><Declaration>struct HasLocalizationKey</Declaration><CommentParts><Abstract><Para>Brief.</Para></Abstract></CommentParts></Class>
 // CHECK87-NEXT: <LocalizationKey>ABC</LocalizationKey>
 
@@ -793,6 +807,9 @@ let strInterpolation = "This is a \(stringStr + "ing") interpolation"
 // CHECK88-NEXT: cursor_info{{$}}
 // CHECK88-NEXT: <Declaration>func hasLocalizationKey2()</Declaration>
 // CHECK88-NEXT: <decl.function.free><syntaxtype.keyword>func</syntaxtype.keyword> <decl.name>hasLocalizationKey2</decl.name>()</decl.function.free>
+// CHECK88-NEXT: DOC COMMENT 
+// CHECK88-NEXT: - LocalizationKey: ABC 
+// CHECK88-NEXT: DOC COMMENT XML 
 // CHECK88-NEXT: <Function file="{{[^"]+}}cursor_info.swift" line="216" column="6"><Name>hasLocalizationKey2()</Name><USR>s:11cursor_info19hasLocalizationKey2yyF</USR><Declaration>func hasLocalizationKey2()</Declaration><CommentParts></CommentParts></Function>
 // CHECK88-NEXT: <LocalizationKey>ABC</LocalizationKey>
 

test/SourceKit/ExtractComment/extract_comments.swift.Comment1.response

@@ -1 +1 @@
-"\nDocComment 1\n\nDocComment 2\n"
+"\nDocComment 1\n\nDocComment 2"

test/SourceKit/ExtractComment/extract_comments.swift.CommentIndent1.response

@@ -1 +1 @@
-"Level1\n  Level2\n    Level3\n      Level4\n        Level5\n"
+"Level1\n  Level2\n    Level3\n      Level4\n        Level5"

test/SourceKit/ExtractComment/extract_comments.swift.CommentIndent2.response

@@ -1 +1 @@
-"Level4\nLevel3\nLevel2\nLevel1\n"
+"Level4\nLevel3\nLevel2\nLevel1"

test/SourceKit/ExtractComment/extract_comments.swift.DocComment1.response

@@ -1 +1 @@
-"DocComment 1\nDocComment 2\nDocComment 3\nDocComment 4\n"
+"DocComment 1\nDocComment 2\nDocComment 3\nDocComment 4"

test/SourceKit/ExtractComment/extract_comments.swift.DocComment2.response

@@ -1 +1 @@
-"DocComment1\nDocComment2\nDocComment3\n"
+"DocComment1\nDocComment2\nDocComment3"

test/SourceKit/ExtractComment/extract_comments.swift.DocComment3.response

@@ -1 +1 @@
-"Line1\nLine2\nLine3\n\t\t\t\n"
+"Line1\nLine2\nLine3\n\t\t\t"

test/SourceKit/ExtractComment/extract_comments.swift.DocCommentEmptyLine1.response

@@ -1 +1 @@
-"Line1\n\nLine2\n"
+"Line1\n\nLine2"

test/SourceKit/ExtractComment/extract_comments.swift.DocCommentEmptyLine2.response

@@ -1 +1 @@
-"Line1\n\nLine2\n"
+"Line1\n\nLine2"

test/SourceKit/ExtractComment/extract_comments.swift.DocCommentEmptyLine3.response

@@ -1 +1 @@
-"Line1\nLine2\n"
+"Line1\nLine2"

tools/SourceKit/include/SourceKit/Core/LangSupport.h

@@ -606,6 +606,7 @@ struct CursorSymbolInfo {
   StringRef TypeUSR;
   StringRef ContainerTypeUSR;
   StringRef DocComment;
+  StringRef DocCommentAsXML;
   StringRef GroupName;
   /// A key for documentation comment localization, if it exists in the doc
   /// comment for the declaration.
@@ -656,6 +657,7 @@ struct CursorSymbolInfo {
     OS << Indentation << "  TypeUSR: " << TypeUSR << '\n';
     OS << Indentation << "  ContainerTypeUSR: " << ContainerTypeUSR << '\n';
     OS << Indentation << "  DocComment: " << DocComment << '\n';
+    OS << Indentation << "  DocCommentAsXML: " << DocCommentAsXML << '\n';
     OS << Indentation << "  GroupName: " << GroupName << '\n';
     OS << Indentation << "  LocalizationKey: " << LocalizationKey << '\n';
     OS << Indentation << "  AnnotatedDeclaration: " << AnnotatedDeclaration
@@ -839,7 +841,7 @@ struct DocEntityInfo {
   llvm::SmallString<64> USR;
   llvm::SmallString<64> OriginalUSR;
   llvm::SmallString<64> ProvideImplementationOfUSR;
-  llvm::SmallString<64> DocComment;
+  llvm::SmallString<64> DocCommentAsXML;
   llvm::SmallString<64> FullyAnnotatedDecl;
   llvm::SmallString<64> FullyAnnotatedGenericSig;
   llvm::SmallString<64> LocalizationKey;

tools/SourceKit/lib/SwiftLang/SwiftDocSupport.cpp

@@ -455,7 +455,7 @@ static bool initDocEntityInfo(const Decl *D,
   }
 
   if (!IsRef) {
-    llvm::raw_svector_ostream OS(Info.DocComment);
+    llvm::raw_svector_ostream OS(Info.DocCommentAsXML);
 
     {
       llvm::SmallString<128> DocBuffer;

tools/SourceKit/lib/SwiftLang/SwiftSourceDocInfo.cpp

@@ -987,9 +987,12 @@ fillSymbolInfo(CursorSymbolInfo &Symbol, const DeclInfo &DInfo,
   }
   Symbol.ContainerTypeUSR = copyAndClearString(Allocator, Buffer);
 
-  ide::getDocumentationCommentAsXML(DInfo.OriginalProperty, OS);
+  ide::getRawDocumentationComment(DInfo.OriginalProperty, OS);
   Symbol.DocComment = copyAndClearString(Allocator, Buffer);
 
+  ide::getDocumentationCommentAsXML(DInfo.OriginalProperty, OS);
+  Symbol.DocCommentAsXML = copyAndClearString(Allocator, Buffer);
+
   {
     auto *Group = DInfo.InSynthesizedExtension ? DInfo.BaseType->getAnyNominal()
                                                : DInfo.VD;

tools/SourceKit/tools/sourcekitd-test/sourcekitd-test.cpp

@@ -1808,6 +1808,7 @@ struct ResponseSymbolInfo {
   const char *TypeUSR = nullptr;
   const char *ContainerTypeUSR = nullptr;
   const char *DocComment = nullptr;
+  const char *DocCommentAsXML = nullptr;
   const char *GroupName = nullptr;
   const char *LocalizationKey = nullptr;
   const char *AnnotatedDeclaration = nullptr;
@@ -1856,6 +1857,8 @@ struct ResponseSymbolInfo {
         sourcekitd_variant_dictionary_get_string(Info, KeyContainerTypeUsr);
 
     Symbol.DocComment =
+        sourcekitd_variant_dictionary_get_string(Info, KeyDocComment);
+    Symbol.DocCommentAsXML =
         sourcekitd_variant_dictionary_get_string(Info, KeyDocFullAsXML);
     Symbol.GroupName =
         sourcekitd_variant_dictionary_get_string(Info, KeyGroupName);
@@ -1981,8 +1984,12 @@ struct ResponseSymbolInfo {
       OS << AnnotatedDeclaration << '\n';
     if (FullyAnnotatedDeclaration)
       OS << FullyAnnotatedDeclaration << '\n';
+    OS << "DOC COMMENT\n";
     if (DocComment)
       OS << DocComment << '\n';
+    OS << "DOC COMMENT XML\n";
+    if (DocCommentAsXML)
+      OS << DocCommentAsXML << '\n';
     if (LocalizationKey) {
       OS << "<LocalizationKey>" << LocalizationKey;
       OS << "</LocalizationKey>" << '\n';

tools/SourceKit/tools/sourcekitd/lib/Service/Requests.cpp

@@ -2451,8 +2451,8 @@ void SKDocConsumer::addDocEntityInfoToDict(const DocEntityInfo &Info,
     Elem.set(KeyIsOptional, Info.IsOptional);
   if (Info.IsAsync)
     Elem.set(KeyIsAsync, Info.IsAsync);
-  if (!Info.DocComment.empty())
-    Elem.set(KeyDocFullAsXML, Info.DocComment);
+  if (!Info.DocCommentAsXML.empty())
+    Elem.set(KeyDocFullAsXML, Info.DocCommentAsXML);
   if (!Info.FullyAnnotatedDecl.empty())
     Elem.set(KeyFullyAnnotatedDecl, Info.FullyAnnotatedDecl);
   if (!Info.FullyAnnotatedGenericSig.empty())
@@ -2604,7 +2604,9 @@ static void addCursorSymbolInfo(const CursorSymbolInfo &Symbol,
   if (!Symbol.ContainerTypeUSR.empty())
     Elem.set(KeyContainerTypeUsr, Symbol.ContainerTypeUSR);
   if (!Symbol.DocComment.empty())
-    Elem.set(KeyDocFullAsXML, Symbol.DocComment);
+    Elem.set(KeyDocComment, Symbol.DocComment);
+  if (!Symbol.DocCommentAsXML.empty())
+    Elem.set(KeyDocFullAsXML, Symbol.DocCommentAsXML);
   if (!Symbol.GroupName.empty())
     Elem.set(KeyGroupName, Symbol.GroupName);
   if (!Symbol.LocalizationKey.empty())

utils/gyb_sourcekit_support/UIDs.py

@@ -35,6 +35,7 @@ UID_KEYS = [
     KEY('GenericParams', 'key.generic_params'),
     KEY('GenericRequirements', 'key.generic_requirements'),
     KEY('DocFullAsXML', 'key.doc.full_as_xml'),
+    KEY('DocComment', 'key.doc_comment'),
     KEY('Line', 'key.line'),
     KEY('Column', 'key.column'),
     KEY('ReceiverUSR', 'key.receiver_usr'),