[Frontend] Switch from YAML to .strings based localization

lib/Frontend/CompilerInvocation.cpp

@@ -1450,7 +1450,7 @@ static bool ParseDiagnosticArgs(DiagnosticOptions &Opts, ArgList &Args,
       // for the specified locale code.
       llvm::SmallString<128> localizationPath(A->getValue());
       llvm::sys::path::append(localizationPath, Opts.LocalizationCode);
-      llvm::sys::path::replace_extension(localizationPath, ".yaml");
+      llvm::sys::path::replace_extension(localizationPath, ".strings");
       if (!llvm::sys::fs::exists(localizationPath)) {
         Diags.diagnose(SourceLoc(), diag::warning_cannot_find_locale_file,
                        Opts.LocalizationCode, localizationPath);

localization/CMakeLists.txt

@@ -6,16 +6,16 @@ add_custom_command(
   COMMAND
     ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR}/diagnostics/ ${CMAKE_BINARY_DIR}/share/swift/diagnostics/
   COMMAND
-    "${SWIFT_NATIVE_SWIFT_TOOLS_PATH}/swift-def-to-yaml-converter"
+    "${SWIFT_NATIVE_SWIFT_TOOLS_PATH}/swift-def-to-strings-converter"
       --output-directory ${CMAKE_BINARY_DIR}/share/swift/diagnostics/
   COMMAND
     "${SWIFT_NATIVE_SWIFT_TOOLS_PATH}/swift-serialize-diagnostics"
-      --input-file-path ${CMAKE_BINARY_DIR}/share/swift/diagnostics/en.yaml
+      --input-file-path ${CMAKE_BINARY_DIR}/share/swift/diagnostics/en.strings
       --output-directory ${CMAKE_BINARY_DIR}/share/swift/diagnostics/
   COMMAND
     ${CMAKE_COMMAND} -E touch ${diagnostic_witness}
   DEPENDS
-    swift-def-to-yaml-converter
+    swift-def-to-strings-converter
     swift-serialize-diagnostics
     # Add files in diagnostics subdirectory when they're created
 )
@@ -31,4 +31,5 @@ swift_install_in_component(
   FILES_MATCHING
   PATTERN "*.db"
   PATTERN "*.yaml"
+  PATTERN "*.strings"
 )

test/diagnostics/Localization/Inputs/en.strings

@@ -0,0 +1,25 @@
+/**
+ *--- en.strings - Localized diagnostic messages for English --------------===*
+ *
+ * This source file is part of the Swift.org open source project
+ *
+ * Copyright (c) 2014 - 2022 Apple Inc. and the Swift project authors
+ * Licensed under Apache License v2.0 with Runtime Library Exception
+ *
+ * See https://swift.org/LICENSE.txt for license information
+ * See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+ *
+ *===----------------------------------------------------------------------===*
+ *
+ * This file defines the diagnostic messages for the English language.
+ * Each diagnostic is described in the following format:
+ *   "<diagnostic-id>" = "<diagnostic-message>";
+ *
+ *===----------------------------------------------------------------------===*
+ *
+ */
+"lex_unterminated_string" = "unterminated string literal";
+"var_init_self_referential" = "variable used within its own initial value";
+/* Tests different number of spaces between id and diagnostic message */
+"cannot_find_in_scope"=     "cannot %select{find|find operator}1 %0 in scope";
+"warning_invalid_locale_code" = "unsupported locale code; supported locale codes are '%0'";

test/diagnostics/Localization/Inputs/en.yaml

@@ -1,30 +0,0 @@
-#===--- en.yaml - Localized diagnostic messages for English ---*- YAML -*-===#
-#
-# This source file is part of the Swift.org open source project
-#
-# Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
-# Licensed under Apache License v2.0 with Runtime Library Exception
-#
-# See https://swift.org/LICENSE.txt for license information
-# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
-#
-#===----------------------------------------------------------------------===#
-#
-# This file defines the diagnostic messages for the English language.
-# Each diagnostic is described in the following format:
-# - id: "<diagnostic-id>"
-#   msg: "<diagnostic-message>"
-# 
-#===----------------------------------------------------------------------===#
-
-- id: "lex_unterminated_string"
-  msg: "unterminated string literal"
-
-- id: "var_init_self_referential"
-  msg: "variable used within its own initial value"
-
-- id: "cannot_find_in_scope"
-  msg: "cannot %select{find|find operator}1 %0 in scope"
-
-- id: "warning_invalid_locale_code"
-  msg: "unsupported locale code; supported locale codes are '%0'"

test/diagnostics/Localization/Inputs/fr.strings

@@ -0,0 +1,27 @@
+/**
+ *===--- fr.strings - Localized diagnostic messages for French ------------===*
+ *
+ * This source file is part of the Swift.org open source project
+ *
+ * Copyright (c) 2014 - 2022 Apple Inc. and the Swift project authors
+ * Licensed under Apache License v2.0 with Runtime Library Exception
+ *
+ * See https://swift.org/LICENSE.txt for license information
+ * See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+ *
+ *===----------------------------------------------------------------------===*
+ *
+ * This file defines the diagnostic messages for the French language.
+ * Each diagnostic is described in the following format:
+ *  "<diagnostic-id>" = "<diagnostic-message>";
+ *
+ */
+
+/* Different order from `en.strings` */
+"var_init_self_referential" = "variable utilisée dans sa propre valeur initiale";
+
+"lex_unterminated_string" = "chaîne non terminée littérale";
+
+"cannot_find_in_scope" = "impossible %select{de trouver|de trouver opérateur}1 %0 portée";
+
+"warning_invalid_locale_code" = "code de paramètres régionaux non pris en charge; les codes pris en charge sont '%0'";

test/diagnostics/Localization/Inputs/fr.yaml

@@ -1,30 +0,0 @@
-#===--- fr.yaml - Localized diagnostic messages for French ---*- YAML -*-===#
-#
-# This source file is part of the Swift.org open source project
-#
-# Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
-# Licensed under Apache License v2.0 with Runtime Library Exception
-#
-# See https://swift.org/LICENSE.txt for license information
-# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
-#
-#===----------------------------------------------------------------------===#
-#
-# This file defines the diagnostic messages for the French language.
-# Each diagnostic is described in the following format:
-# - id: "<diagnostic-id>"
-#   msg: "<diagnostic-message>"
-# 
-#===----------------------------------------------------------------------===#
-
-- id: "lex_unterminated_string"
-  msg: "chaîne non terminée littérale"
-
-- id: "var_init_self_referential"
-  msg: "variable utilisée dans sa propre valeur initiale"
-
-- id: "cannot_find_in_scope"
-  msg: "impossible %select{de trouver|de trouver opérateur}1 %0 portée"
-
-- id: "warning_invalid_locale_code"
-  msg: "code de paramètres régionaux non pris en charge; les codes pris en charge sont '%0'"

test/diagnostics/Localization/fr_debug_diagnostic_name.swift

@@ -1,6 +1,6 @@
 // RUN: %empty-directory(%t)
-// RUN: swift-serialize-diagnostics --input-file-path=%S/Inputs/fr.yaml --output-directory=%t/
-// RUN: swift-serialize-diagnostics --input-file-path=%S/Inputs/en.yaml --output-directory=%t/
+// RUN: swift-serialize-diagnostics --input-file-path=%S/Inputs/fr.strings --output-directory=%t/
+// RUN: swift-serialize-diagnostics --input-file-path=%S/Inputs/en.strings --output-directory=%t/
 // RUN: not %target-swift-frontend -debug-diagnostic-names -localization-path %S/Inputs -locale fr -typecheck %s 2>&1 | %FileCheck %s --check-prefix=CHECK_NAMES
 
 _ = "HI!

test/diagnostics/Localization/fr_localization.swift

@@ -1,6 +1,6 @@
 // RUN: %empty-directory(%t)
-// RUN: swift-serialize-diagnostics --input-file-path=%S/Inputs/fr.yaml --output-directory=%t/
-// RUN: swift-serialize-diagnostics --input-file-path=%S/Inputs/en.yaml --output-directory=%t/
+// RUN: swift-serialize-diagnostics --input-file-path=%S/Inputs/fr.strings --output-directory=%t/
+// RUN: swift-serialize-diagnostics --input-file-path=%S/Inputs/en.strings --output-directory=%t/
 // RUN: %target-typecheck-verify-swift -localization-path %t -locale fr
 
 _ = "HI!

test/diagnostics/Localization/no_localization_files_and_wrong_path.swift

@@ -1,6 +1,6 @@
 // RUN: %target-typecheck-verify-swift -localization-path /Nonexistent_path -locale en
 
-// <unknown>:0: warning: cannot find translations for 'en' at '/Nonexistent_path/en.yaml': no such file
+// <unknown>:0: warning: cannot find translations for 'en' at '/Nonexistent_path/en.strings': no such file
 // <unknown>:0: warning: specified localization directory '/Nonexistent_path' does not exist, translation is disabled
 
 _ = "HI!

tools/swift-serialize-diagnostics/swift-serialize-diagnostics.cpp

@@ -42,7 +42,7 @@ static llvm::cl::OptionCategory Category("swift-serialize-diagnostics Options");
 
 static llvm::cl::opt<std::string>
     InputFilePath("input-file-path",
-                  llvm::cl::desc("Path to the YAML input file"),
+                  llvm::cl::desc("Path to the YAML or `.strings` input file"),
                   llvm::cl::cat(Category));
 
 static llvm::cl::opt<std::string>
@@ -60,22 +60,44 @@ int main(int argc, char *argv[]) {
                                     "Swift Serialize Diagnostics Tool\n");
 
   if (!llvm::sys::fs::exists(options::InputFilePath)) {
-    llvm::errs() << "YAML file not found\n";
+    llvm::errs() << "diagnostics file not found\n";
     return EXIT_FAILURE;
   }
 
-  YAMLLocalizationProducer yaml(options::InputFilePath);
-
   auto localeCode = llvm::sys::path::filename(options::InputFilePath);
   llvm::SmallString<128> SerializedFilePath(options::OutputDirectory);
   llvm::sys::path::append(SerializedFilePath, localeCode);
   llvm::sys::path::replace_extension(SerializedFilePath, ".db");
 
   SerializedLocalizationWriter Serializer;
-  yaml.forEachAvailable(
-      [&Serializer](swift::DiagID id, llvm::StringRef translation) {
-        Serializer.insert(id, translation);
-      });
+
+  if (llvm::sys::path::extension(options::InputFilePath) == ".yaml") {
+    YAMLLocalizationProducer yaml(options::InputFilePath);
+
+    yaml.forEachAvailable(
+        [&Serializer](swift::DiagID id, llvm::StringRef translation) {
+          Serializer.insert(id, translation);
+        });
+
+    // Print out the diagnostics IDs that are available in YAML but not
+    // available in `.def`
+    if (!yaml.unknownIDs.empty()) {
+      llvm::errs() << "These diagnostic IDs are no longer available: '";
+      llvm::interleave(
+          yaml.unknownIDs, [&](std::string id) { llvm::errs() << id; },
+          [&] { llvm::errs() << ", "; });
+      llvm::errs() << "'\n";
+    }
+  } else {
+    assert(llvm::sys::path::extension(options::InputFilePath) == ".strings");
+
+    StringsLocalizationProducer strings(options::InputFilePath);
+
+    strings.forEachAvailable(
+        [&Serializer](swift::DiagID id, llvm::StringRef translation) {
+          Serializer.insert(id, translation);
+        });
+  }
 
   if (Serializer.emit(SerializedFilePath.str())) {
     llvm::errs() << "Cannot serialize diagnostic file "
@@ -83,15 +105,5 @@ int main(int argc, char *argv[]) {
     return EXIT_FAILURE;
   }
 
-  // Print out the diagnostics IDs that are available in YAML but not available
-  // in `.def`
-  if (!yaml.unknownIDs.empty()) {
-    llvm::errs() << "These diagnostic IDs are no longer available: '";
-    llvm::interleave(
-        yaml.unknownIDs, [&](std::string id) { llvm::errs() << id; },
-        [&] { llvm::errs() << ", "; });
-    llvm::errs() << "'\n";
-  }
-
   return EXIT_SUCCESS;
 }

unittests/Localization/CMakeLists.txt

@@ -1,5 +1,5 @@
 add_swift_unittest(swiftLocalizationTests
-  DefToYAMLConverterTests.cpp
+  DefToStringsConverterTests.cpp
   SerializationTests.cpp)
 
 target_link_libraries(swiftLocalizationTests

unittests/Localization/DefToStringsConverterTests.cpp

@@ -1,8 +1,8 @@
-//===--- DefToYAMLConverterTests.cpp -------------------------------------===//
+//===--- DefToStringsConverterTests.cpp -----------------------------------===//
 //
 // This source file is part of the Swift.org open source project
 //
-// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
+// Copyright (c) 2014 - 2022 Apple Inc. and the Swift project authors
 // Licensed under Apache License v2.0 with Runtime Library Exception
 //
 // See https://swift.org/LICENSE.txt for license information
@@ -49,22 +49,22 @@ TEST_F(LocalizationTest, MissingLocalizationFiles) {
   ASSERT_TRUE(llvm::sys::fs::exists(getDefaultLocalizationPath()));
   llvm::SmallString<128> EnglishLocalization(getDefaultLocalizationPath());
   llvm::sys::path::append(EnglishLocalization, "en");
-  llvm::sys::path::replace_extension(EnglishLocalization, ".yaml");
+  llvm::sys::path::replace_extension(EnglishLocalization, ".strings");
   ASSERT_TRUE(llvm::sys::fs::exists(EnglishLocalization));
   llvm::sys::path::replace_extension(EnglishLocalization, ".db");
   ASSERT_TRUE(llvm::sys::fs::exists(EnglishLocalization));
 }
 
 TEST_F(LocalizationTest, ConverterTestMatchDiagnosticMessagesSequentially) {
-  YAMLLocalizationProducer yaml(YAMLPath);
-  yaml.forEachAvailable([](swift::DiagID id, llvm::StringRef translation) {
+  StringsLocalizationProducer strings(DiagsPath);
+  strings.forEachAvailable([](swift::DiagID id, llvm::StringRef translation) {
     llvm::StringRef msg = diagnosticMessages[static_cast<uint32_t>(id)];
     ASSERT_EQ(msg, translation);
   });
 }
 
 TEST_F(LocalizationTest, ConverterTestMatchDiagnosticMessagesRandomly) {
-  YAMLLocalizationProducer yaml(YAMLPath);
+  StringsLocalizationProducer strings(DiagsPath);
 
   std::random_device rd;
   std::mt19937 gen(rd());
@@ -74,7 +74,7 @@ TEST_F(LocalizationTest, ConverterTestMatchDiagnosticMessagesRandomly) {
     unsigned randomNum = RandNumber(LocalDiagID::NumDiags);
     DiagID randomId = static_cast<DiagID>(randomNum);
     llvm::StringRef msg = diagnosticMessages[randomNum];
-    llvm::StringRef translation = yaml.getMessageOr(randomId, "");
+    llvm::StringRef translation = strings.getMessageOr(randomId, "");
     ASSERT_EQ(msg, translation);
   }
 }

unittests/Localization/LocalizationTest.h

@@ -53,15 +53,15 @@ struct LocalizationTest : public ::testing::Test {
   llvm::SmallVector<std::string, 4> TempFiles;
 
 public:
-  std::string YAMLPath;
+  std::string DiagsPath;
 
   LocalizationTest() {
-    YAMLPath = std::string(createTemporaryFile("en", "yaml"));
+    DiagsPath = std::string(createTemporaryFile("en", "strings"));
   }
 
   void SetUp() override {
-    bool failed = convertDefIntoYAML(YAMLPath);
-    assert(!failed && "failed to generate a YAML file");
+    bool failed = convertDefIntoStrings(DiagsPath);
+    assert(!failed && "failed to generate a `.strings` file");
   }
 
   void TearDown() override {
@@ -85,7 +85,7 @@ public:
   unsigned RandNumber(unsigned n) { return unsigned(rand()) % n; }
 
 protected:
-  static bool convertDefIntoYAML(std::string outputPath) {
+  static bool convertDefIntoStrings(std::string outputPath) {
     std::error_code error;
     llvm::raw_fd_ostream OS(outputPath, error, llvm::sys::fs::OF_None);
     if (OS.has_error() || error)
@@ -95,7 +95,7 @@ protected:
     llvm::ArrayRef<const char *> messages(diagnosticMessages,
                                           LocalDiagID::NumDiags);
 
-    DefToYAMLConverter converter(ids, messages);
+    DefToStringsConverter converter(ids, messages);
     converter.convert(OS);
 
     OS.flush();

unittests/Localization/SerializationTests.cpp

@@ -24,8 +24,8 @@
 using namespace swift::diag;
 using namespace swift::unittests;
 
-TEST_F(LocalizationTest, TestYAMLSerialization) {
-  YAMLLocalizationProducer yaml(YAMLPath);
+TEST_F(LocalizationTest, TestStringsSerialization) {
+  StringsLocalizationProducer strings(DiagsPath);
 
   auto dbFile = createTemporaryFile("en", "db");
 
@@ -33,9 +33,10 @@ TEST_F(LocalizationTest, TestYAMLSerialization) {
   {
     SerializedLocalizationWriter writer;
 
-    yaml.forEachAvailable([&writer](swift::DiagID id, llvm::StringRef translation) {
-                            writer.insert(id, translation);
-                          });
+    strings.forEachAvailable(
+        [&writer](swift::DiagID id, llvm::StringRef translation) {
+          writer.insert(id, translation);
+        });
 
     ASSERT_FALSE(writer.emit(dbFile));
   }
@@ -45,9 +46,10 @@ TEST_F(LocalizationTest, TestYAMLSerialization) {
   ASSERT_TRUE(dbContent);
 
   SerializedLocalizationProducer db(std::move(dbContent.get()));
-  yaml.forEachAvailable([&db](swift::DiagID id, llvm::StringRef translation) {
-    ASSERT_EQ(translation, db.getMessageOr(id, "<no-fallback>"));
-  });
+  strings.forEachAvailable(
+      [&db](swift::DiagID id, llvm::StringRef translation) {
+        ASSERT_EQ(translation, db.getMessageOr(id, "<no-fallback>"));
+      });
 }
 
 TEST_F(LocalizationTest, TestSerializationOfEmptyFile) {
@@ -55,7 +57,7 @@ TEST_F(LocalizationTest, TestSerializationOfEmptyFile) {
   SerializedLocalizationWriter writer;
   ASSERT_FALSE(writer.emit(dbFile));
 
-  YAMLLocalizationProducer yaml(YAMLPath);
+  StringsLocalizationProducer strings(DiagsPath);
 
   // Reading of the empty `db` file should always return default message.
   {
@@ -63,7 +65,8 @@ TEST_F(LocalizationTest, TestSerializationOfEmptyFile) {
     ASSERT_TRUE(dbContent);
 
     SerializedLocalizationProducer db(std::move(dbContent.get()));
-    yaml.forEachAvailable([&db](swift::DiagID id, llvm::StringRef translation) {
+    strings.forEachAvailable([&db](swift::DiagID id,
+                                   llvm::StringRef translation) {
       ASSERT_EQ("<<<default-fallback>>>",
                 db.getMessageOr(id, "<<<default-fallback>>>"));
     });
@@ -80,16 +83,17 @@ TEST_F(LocalizationTest, TestSerializationWithGaps) {
     includedMessages.flip(position);
   }
 
-  YAMLLocalizationProducer yaml(YAMLPath);
+  StringsLocalizationProducer strings(DiagsPath);
   auto dbFile = createTemporaryFile("en", "db");
 
   {
     SerializedLocalizationWriter writer;
 
-    yaml.forEachAvailable([&](swift::DiagID id, llvm::StringRef translation) {
-      if (includedMessages.test((unsigned)id))
-        writer.insert(id, translation);
-    });
+    strings.forEachAvailable(
+        [&](swift::DiagID id, llvm::StringRef translation) {
+          if (includedMessages.test((unsigned)id))
+            writer.insert(id, translation);
+        });
 
     ASSERT_FALSE(writer.emit(dbFile));
   }
@@ -100,7 +104,8 @@ TEST_F(LocalizationTest, TestSerializationWithGaps) {
     ASSERT_TRUE(dbContent);
 
     SerializedLocalizationProducer db(std::move(dbContent.get()));
-    yaml.forEachAvailable([&](swift::DiagID id, llvm::StringRef translation) {
+    strings.forEachAvailable([&](swift::DiagID id,
+                                 llvm::StringRef translation) {
       auto position = (unsigned)id;
 
       std::string expectedMessage = includedMessages.test(position)