[SourceKit] Vend the localization key found in documentation comments

If a documentation comment has a - LocalizationKey: field, strip it
out of the documentation body and report it in cursor/doc info with
the key "key.localization_key".

rdar://problem/30383329

include/swift/IDE/CommentConversion.h

@@ -29,6 +29,10 @@ namespace ide {
 /// \returns true if the declaration has a documentation comment.
 bool getDocumentationCommentAsXML(const Decl *D, raw_ostream &OS);
 
+/// If the declaration has a documentation comment and a localization key,
+/// print it into the given output stream and return true. Else, return false.
+bool getLocalizationKey(const Decl *D, raw_ostream &OS);
+
 /// Converts the given comment to Doxygen.
 void getDocumentationCommentAsDoxygen(const DocComment *DC, raw_ostream &OS);
 

include/swift/Markup/AST.h

@@ -28,6 +28,7 @@ class ParamField;
 class ReturnsField;
 class TagField;
 class ThrowsField;
+class LocalizationKeyField;
 
 /// The basic structure of a doc comment attached to a Swift
 /// declaration.

include/swift/Markup/ASTNodes.def

@@ -61,7 +61,7 @@ MARKUP_AST_NODE(PrivateExtension, MarkupASTNode)
   MARKUP_AST_NODE(ExperimentField, PrivateExtension)
   MARKUP_AST_NODE(ImportantField, PrivateExtension)
   MARKUP_AST_NODE(InvariantField, PrivateExtension)
-  MARKUP_AST_NODE(LocalizationKey, PrivateExtension)
+  MARKUP_AST_NODE(LocalizationKeyField, PrivateExtension)
   MARKUP_AST_NODE(MutatingvariantField, PrivateExtension)
   MARKUP_AST_NODE(NonmutatingvariantField, PrivateExtension)
   MARKUP_AST_NODE(NoteField, PrivateExtension)

lib/IDE/CommentConversion.cpp

@@ -443,6 +443,20 @@ bool ide::getDocumentationCommentAsXML(const Decl *D, raw_ostream &OS) {
   return true;
 }
 
+bool ide::getLocalizationKey(const Decl *D, raw_ostream &OS) {
+  swift::markup::MarkupContext MC;
+  auto DC = getCascadingDocComment(MC, D);
+  if (!DC.hasValue())
+    return false;
+
+  if (const auto LKF = DC.getValue()->getLocalizationKeyField()) {
+    printInlinesUnder(LKF.getValue(), OS);
+    return true;
+  }
+
+  return false;
+}
+
 //===----------------------------------------------------------------------===//
 // Conversion to Doxygen.
 //===----------------------------------------------------------------------===//

test/Inputs/comment_to_something_conversion.swift

@@ -467,3 +467,13 @@ public func codeListingWithDefaultLanguage() {}
 /// ```
 public func codeListingWithOtherLanguage() {}
 // CHECK: DocCommentAsXML=[<Function file="{{.*}}" line="{{.*}}" column="{{.*}}"><Name>codeListingWithOtherLanguage()</Name><USR>s:14swift_ide_test28codeListingWithOtherLanguageyyF</USR><Declaration>public func codeListingWithOtherLanguage()</Declaration><Abstract><Para>Brief.</Para></Abstract><Discussion><CodeListing language="c++"><zCodeLineNumbered><![CDATA[Something::Something::create();]]></zCodeLineNumbered><zCodeLineNumbered></zCodeLineNumbered></CodeListing></Discussion></Function>]
+
+/// Brief.
+///
+/// - LocalizationKey: ABC
+public func localizationKeyShouldNotAppearInDocComments() {}
+// CHECK: DocCommentAsXML=[<Function file="{{.*}}" line="{{.*}}" column="{{.*}}"><Name>localizationKeyShouldNotAppearInDocComments()</Name><USR>s:14swift_ide_test43localizationKeyShouldNotAppearInDocCommentsyyF</USR><Declaration>public func localizationKeyShouldNotAppearInDocComments()</Declaration><Abstract><Para>Brief.</Para></Abstract></Function>]
+
+/// - LocalizationKey: ABC
+public func localizationKeyShouldNotAppearInDocComments2() {}
+// CHECK: DocCommentAsXML=[<Function file="{{.*}}" line="{{.*}}" column="{{.*}}"><Name>localizationKeyShouldNotAppearInDocComments2()</Name><USR>s:14swift_ide_test44localizationKeyShouldNotAppearInDocComments2yyF</USR><Declaration>public func localizationKeyShouldNotAppearInDocComments2()</Declaration></Function>]

test/SourceKit/CursorInfo/cursor_info.swift

@@ -206,6 +206,14 @@ func convention5(_: @convention(method) ()->()) {}
 func convention6(_: @convention(objc_method) ()->()) {}
 func convention7(_: @convention(witness_method) ()->()) {}
 
+/// Brief.
+///
+/// - LocalizationKey: ABC
+struct HasLocalizationKey {}
+
+/// - LocalizationKey: ABC
+func hasLocalizationKey2() {}
+
 // RUN: rm -rf %t.tmp
 // RUN: mkdir -p %t.tmp
 // RUN: %swiftc_driver -emit-module -o %t.tmp/FooSwiftModule.swiftmodule %S/Inputs/FooSwiftModule.swift
@@ -702,3 +710,25 @@ func convention7(_: @convention(witness_method) ()->()) {}
 // RUN: %sourcekitd-test -req=cursor -pos=206:6 %s -- -F %S/../Inputs/libIDE-mock-sdk -I %t.tmp %mcp_opt %s | %FileCheck -check-prefix=CHECK86 %s
 // RUN: %sourcekitd-test -req=cursor -pos=207:6 %s -- -F %S/../Inputs/libIDE-mock-sdk -I %t.tmp %mcp_opt %s | %FileCheck -check-prefix=CHECK86 %s
 // CHECK86: <syntaxtype.attribute.builtin><syntaxtype.attribute.name>@convention</syntaxtype.attribute.name>({{[a-z_]*}})</syntaxtype.attribute.builtin>
+
+// RUN: %sourcekitd-test -req=cursor -pos=212:8 %s -- -F %S/../Inputs/libIDE-mock-sdk -I %t.tmp %mcp_opt %s | %FileCheck -check-prefix=CHECK87 %s
+// CHECK87:      source.lang.swift.decl.struct (212:8-212:26)
+// CHECK87-NEXT: HasLocalizationKey
+// CHECK87-NEXT: s:V11cursor_info18HasLocalizationKey
+// CHECK87-NEXT: HasLocalizationKey.Type
+// CHECK87-NEXT: _Tt
+// CHECK87-NEXT: <Declaration>struct HasLocalizationKey</Declaration>
+// CHECK87-NEXT: <decl.struct><syntaxtype.keyword>struct</syntaxtype.keyword> <decl.name>HasLocalizationKey</decl.name></decl.struct>
+// CHECK87-NEXT: <Class file="{{[^"]+}}cursor_info.swift" line="212" column="8"><Name>HasLocalizationKey</Name><USR>s:V11cursor_info18HasLocalizationKey</USR><Declaration>struct HasLocalizationKey</Declaration><Abstract><Para>Brief.</Para></Abstract></Class>
+// CHECK87-NEXT: <LocalizationKey>ABC</LocalizationKey>
+
+// RUN: %sourcekitd-test -req=cursor -pos=215:6 %s -- -F %S/../Inputs/libIDE-mock-sdk -I %t.tmp %mcp_opt %s | %FileCheck -check-prefix=CHECK88 %s
+// CHECK88:      source.lang.swift.decl.function.free (215:6-215:27)
+// CHECK88-NEXT: hasLocalizationKey2
+// CHECK88-NEXT: s:F11cursor_info19hasLocalizationKey2FT_T_
+// CHECK88-NEXT: () -> ()
+// CHECK88-NEXT: _Tt
+// CHECK88-NEXT: <Declaration>func hasLocalizationKey2()</Declaration>
+// CHECK88-NEXT: <decl.function.free><syntaxtype.keyword>func</syntaxtype.keyword> <decl.name>hasLocalizationKey2</decl.name>()</decl.function.free>
+// CHECK88-NEXT: <Function file="{{[^"]+}}cursor_info.swift" line="215" column="6"><Name>hasLocalizationKey2()</Name><USR>s:F11cursor_info19hasLocalizationKey2FT_T_</USR><Declaration>func hasLocalizationKey2()</Declaration></Function
+// CHECK88-NEXT: <LocalizationKey>ABC</LocalizationKey>

tools/SourceKit/include/SourceKit/Core/LangSupport.h

@@ -258,6 +258,9 @@ struct CursorInfo {
   StringRef DocComment;
   StringRef TypeInterface;
   StringRef GroupName;
+  /// A key for documentation comment localization, if it exists in the doc
+  /// comment for the declaration.
+  StringRef LocalizationKey;
   /// Annotated XML pretty printed declaration.
   StringRef AnnotatedDeclaration;
   /// Fully annotated XML pretty printed declaration.
@@ -330,6 +333,7 @@ struct DocEntityInfo {
   llvm::SmallString<64> ProvideImplementationOfUSR;
   llvm::SmallString<64> DocComment;
   llvm::SmallString<64> FullyAnnotatedDecl;
+  llvm::SmallString<64> LocalizationKey;
   std::vector<DocGenericParam> GenericParams;
   std::vector<std::string> GenericRequirements;
   unsigned Offset = 0;

tools/SourceKit/lib/SwiftLang/SwiftDocSupport.cpp

@@ -378,6 +378,9 @@ static bool initDocEntityInfo(const Decl *D, const Decl *SynthesizedTarget,
 
     initDocGenericParams(D, Info);
 
+    llvm::raw_svector_ostream LocalizationKeyOS(Info.LocalizationKey);
+    ide::getLocalizationKey(D, LocalizationKeyOS);
+
     if (auto *VD = dyn_cast<ValueDecl>(D)) {
       llvm::raw_svector_ostream OS(Info.FullyAnnotatedDecl);
       if (SynthesizedTarget)

tools/SourceKit/lib/SwiftLang/SwiftSourceDocInfo.cpp

@@ -715,7 +715,17 @@ static bool passCursorInfoForDecl(const ValueDecl *VD,
   }
   unsigned GroupEnd = SS.size();
 
+  unsigned LocalizationBegin = SS.size();
+  {
+    llvm::raw_svector_ostream OS(SS);
+    ide::getLocalizationKey(VD, OS);
+  }
+  unsigned LocalizationEnd = SS.size();
+
   DelayedStringRetriever OverUSRsStream(SS);
+
+  SmallVector<std::pair<unsigned, unsigned>, 4> OverUSROffs;
+
   ide::walkOverriddenDecls(VD,
     [&](llvm::PointerUnion<const ValueDecl*, const clang::NamedDecl*> D) {
       OverUSRsStream.startPiece();
@@ -798,6 +808,8 @@ static bool passCursorInfoForDecl(const ValueDecl *VD,
   StringRef FullyAnnotatedDecl =
       StringRef(SS.begin() + FullDeclBegin, FullDeclEnd - FullDeclBegin);
   StringRef GroupName = StringRef(SS.begin() + GroupBegin, GroupEnd - GroupBegin);
+  StringRef LocalizationKey = StringRef(SS.begin() + LocalizationBegin,
+                                        LocalizationEnd - LocalizationBegin);
 
   llvm::Optional<std::pair<unsigned, unsigned>> DeclarationLoc;
   StringRef Filename;
@@ -837,6 +849,7 @@ static bool passCursorInfoForDecl(const ValueDecl *VD,
   Info.OverrideUSRs = OverUSRs;
   Info.AnnotatedRelatedDeclarations = AnnotatedRelatedDecls;
   Info.GroupName = GroupName;
+  Info.LocalizationKey = LocalizationKey;
   Info.IsSystem = IsSystem;
   Info.TypeInterface = StringRef();
   Receiver(Info);

tools/SourceKit/tools/sourcekitd-test/sourcekitd-test.cpp

@@ -92,6 +92,7 @@ static sourcekitd_uid_t KeyOffset;
 static sourcekitd_uid_t KeySourceFile;
 static sourcekitd_uid_t KeyModuleName;
 static sourcekitd_uid_t KeyGroupName;
+static sourcekitd_uid_t KeyLocalizationKey;
 static sourcekitd_uid_t KeyActionName;
 static sourcekitd_uid_t KeySynthesizedExtension;
 static sourcekitd_uid_t KeyName;
@@ -215,6 +216,7 @@ static int skt_main(int argc, const char **argv) {
   KeySourceFile = sourcekitd_uid_get_from_cstr("key.sourcefile");
   KeyModuleName = sourcekitd_uid_get_from_cstr("key.modulename");
   KeyGroupName = sourcekitd_uid_get_from_cstr("key.groupname");
+  KeyLocalizationKey = sourcekitd_uid_get_from_cstr("key.localization_key");
   KeyActionName = sourcekitd_uid_get_from_cstr("key.actionname");
   KeySynthesizedExtension = sourcekitd_uid_get_from_cstr("key.synthesizedextensions");
   KeyName = sourcekitd_uid_get_from_cstr("key.name");
@@ -1191,6 +1193,9 @@ static void printCursorInfo(sourcekitd_variant_t Info, StringRef FilenameIn,
                                                               KeyModuleName);
   const char *GroupName = sourcekitd_variant_dictionary_get_string(Info,
                                                                    KeyGroupName);
+
+  const char *LocalizationKey =
+    sourcekitd_variant_dictionary_get_string(Info, KeyLocalizationKey);
   const char *ModuleInterfaceName =
       sourcekitd_variant_dictionary_get_string(Info, KeyModuleInterfaceName);
   const char *TypeInterface =
@@ -1287,6 +1292,9 @@ static void printCursorInfo(sourcekitd_variant_t Info, StringRef FilenameIn,
     OS << FullAnnotDecl << '\n';
   if (DocFullAsXML)
     OS << DocFullAsXML << '\n';
+  if (LocalizationKey)
+    OS << "<LocalizationKey>" << LocalizationKey;
+  OS << "</LocalizationKey>" << '\n';
   OS << "OVERRIDES BEGIN\n";
   for (auto OverUSR : OverrideUSRs)
     OS << OverUSR << '\n';

tools/SourceKit/tools/sourcekitd/lib/API/DictionaryKeys.h

@@ -129,6 +129,7 @@ extern SourceKit::UIdent KeyBaseName;
 extern SourceKit::UIdent KeyArgNames;
 extern SourceKit::UIdent KeySelectorPieces;
 extern SourceKit::UIdent KeyNameKind;
+extern SourceKit::UIdent KeyLocalizationKey;
 
 /// \brief Used for determining the printing order of dictionary keys.
 bool compareDictKeys(SourceKit::UIdent LHS, SourceKit::UIdent RHS);

tools/SourceKit/tools/sourcekitd/lib/API/Requests.cpp

@@ -1209,6 +1209,8 @@ void SKDocConsumer::addDocEntityInfoToDict(const DocEntityInfo &Info,
     Elem.set(KeyDocFullAsXML, Info.DocComment);
   if (!Info.FullyAnnotatedDecl.empty())
     Elem.set(KeyFullyAnnotatedDecl, Info.FullyAnnotatedDecl);
+  if (!Info.LocalizationKey.empty())
+    Elem.set(KeyLocalizationKey, Info.LocalizationKey);
 
   if (!Info.GenericParams.empty()) {
     auto GPArray = Elem.setArray(KeyGenericParams);
@@ -1384,6 +1386,8 @@ static void reportCursorInfo(const CursorInfo &Info, ResponseReceiver Rec) {
     Elem.set(KeyModuleName, Info.ModuleName);
   if (!Info.GroupName.empty())
     Elem.set(KeyGroupName, Info.GroupName);
+  if (!Info.LocalizationKey.empty())
+    Elem.set(KeyLocalizationKey, Info.LocalizationKey);
   if (!Info.ModuleInterfaceName.empty())
     Elem.set(KeyModuleInterfaceName, Info.ModuleInterfaceName);
   if (Info.DeclarationLoc.hasValue()) {

tools/SourceKit/tools/sourcekitd/lib/API/sourcekitdAPI-Common.cpp

@@ -142,6 +142,7 @@ UIdent sourcekitd::KeyBaseName("key.basename");
 UIdent sourcekitd::KeyArgNames("key.argnames");
 UIdent sourcekitd::KeySelectorPieces("key.selectorpieces");
 UIdent sourcekitd::KeyNameKind("key.namekind");
+UIdent sourcekitd::KeyLocalizationKey("key.localization_key");
 
 /// \brief Order for the keys to use when emitting the debug description of
 /// dictionaries.