From 5a164c5f8aaaa1e5794db72200fc95a9408c6f39 Mon Sep 17 00:00:00 2001 From: Pavel Yaskevich Date: Mon, 22 Aug 2022 10:22:41 -0700 Subject: [PATCH] [Frontend] Switch from YAML to `.strings` based localization --- lib/Frontend/CompilerInvocation.cpp | 2 +- localization/CMakeLists.txt | 7 +-- .../Localization/Inputs/en.strings | 25 ++++++++++ test/diagnostics/Localization/Inputs/en.yaml | 30 ------------ .../Localization/Inputs/fr.strings | 27 +++++++++++ test/diagnostics/Localization/Inputs/fr.yaml | 30 ------------ .../fr_debug_diagnostic_name.swift | 4 +- .../Localization/fr_localization.swift | 4 +- ...no_localization_files_and_wrong_path.swift | 2 +- .../swift-serialize-diagnostics.cpp | 48 ++++++++++++------- unittests/Localization/CMakeLists.txt | 2 +- ...sts.cpp => DefToStringsConverterTests.cpp} | 14 +++--- unittests/Localization/LocalizationTest.h | 12 ++--- unittests/Localization/SerializationTests.cpp | 37 +++++++------- 14 files changed, 127 insertions(+), 117 deletions(-) create mode 100644 test/diagnostics/Localization/Inputs/en.strings delete mode 100644 test/diagnostics/Localization/Inputs/en.yaml create mode 100644 test/diagnostics/Localization/Inputs/fr.strings delete mode 100644 test/diagnostics/Localization/Inputs/fr.yaml rename unittests/Localization/{DefToYAMLConverterTests.cpp => DefToStringsConverterTests.cpp} (85%) diff --git a/lib/Frontend/CompilerInvocation.cpp b/lib/Frontend/CompilerInvocation.cpp index f52fe884217..83490d354e3 100644 --- a/lib/Frontend/CompilerInvocation.cpp +++ b/lib/Frontend/CompilerInvocation.cpp @@ -1450,7 +1450,7 @@ static bool ParseDiagnosticArgs(DiagnosticOptions &Opts, ArgList &Args, // for the specified locale code. llvm::SmallString<128> localizationPath(A->getValue()); llvm::sys::path::append(localizationPath, Opts.LocalizationCode); - llvm::sys::path::replace_extension(localizationPath, ".yaml"); + llvm::sys::path::replace_extension(localizationPath, ".strings"); if (!llvm::sys::fs::exists(localizationPath)) { Diags.diagnose(SourceLoc(), diag::warning_cannot_find_locale_file, Opts.LocalizationCode, localizationPath); diff --git a/localization/CMakeLists.txt b/localization/CMakeLists.txt index 07a3585a66c..537303164ea 100644 --- a/localization/CMakeLists.txt +++ b/localization/CMakeLists.txt @@ -6,16 +6,16 @@ add_custom_command( COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR}/diagnostics/ ${CMAKE_BINARY_DIR}/share/swift/diagnostics/ COMMAND - "${SWIFT_NATIVE_SWIFT_TOOLS_PATH}/swift-def-to-yaml-converter" + "${SWIFT_NATIVE_SWIFT_TOOLS_PATH}/swift-def-to-strings-converter" --output-directory ${CMAKE_BINARY_DIR}/share/swift/diagnostics/ COMMAND "${SWIFT_NATIVE_SWIFT_TOOLS_PATH}/swift-serialize-diagnostics" - --input-file-path ${CMAKE_BINARY_DIR}/share/swift/diagnostics/en.yaml + --input-file-path ${CMAKE_BINARY_DIR}/share/swift/diagnostics/en.strings --output-directory ${CMAKE_BINARY_DIR}/share/swift/diagnostics/ COMMAND ${CMAKE_COMMAND} -E touch ${diagnostic_witness} DEPENDS - swift-def-to-yaml-converter + swift-def-to-strings-converter swift-serialize-diagnostics # Add files in diagnostics subdirectory when they're created ) @@ -31,4 +31,5 @@ swift_install_in_component( FILES_MATCHING PATTERN "*.db" PATTERN "*.yaml" + PATTERN "*.strings" ) diff --git a/test/diagnostics/Localization/Inputs/en.strings b/test/diagnostics/Localization/Inputs/en.strings new file mode 100644 index 00000000000..2ca51b4296a --- /dev/null +++ b/test/diagnostics/Localization/Inputs/en.strings @@ -0,0 +1,25 @@ +/** + *--- en.strings - Localized diagnostic messages for English --------------===* + * + * This source file is part of the Swift.org open source project + * + * Copyright (c) 2014 - 2022 Apple Inc. and the Swift project authors + * Licensed under Apache License v2.0 with Runtime Library Exception + * + * See https://swift.org/LICENSE.txt for license information + * See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors + * + *===----------------------------------------------------------------------===* + * + * This file defines the diagnostic messages for the English language. + * Each diagnostic is described in the following format: + * "" = ""; + * + *===----------------------------------------------------------------------===* + * + */ +"lex_unterminated_string" = "unterminated string literal"; +"var_init_self_referential" = "variable used within its own initial value"; +/* Tests different number of spaces between id and diagnostic message */ +"cannot_find_in_scope"= "cannot %select{find|find operator}1 %0 in scope"; +"warning_invalid_locale_code" = "unsupported locale code; supported locale codes are '%0'"; diff --git a/test/diagnostics/Localization/Inputs/en.yaml b/test/diagnostics/Localization/Inputs/en.yaml deleted file mode 100644 index b85686db42c..00000000000 --- a/test/diagnostics/Localization/Inputs/en.yaml +++ /dev/null @@ -1,30 +0,0 @@ -#===--- en.yaml - Localized diagnostic messages for English ---*- YAML -*-===# -# -# This source file is part of the Swift.org open source project -# -# Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors -# Licensed under Apache License v2.0 with Runtime Library Exception -# -# See https://swift.org/LICENSE.txt for license information -# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -# -#===----------------------------------------------------------------------===# -# -# This file defines the diagnostic messages for the English language. -# Each diagnostic is described in the following format: -# - id: "" -# msg: "" -# -#===----------------------------------------------------------------------===# - -- id: "lex_unterminated_string" - msg: "unterminated string literal" - -- id: "var_init_self_referential" - msg: "variable used within its own initial value" - -- id: "cannot_find_in_scope" - msg: "cannot %select{find|find operator}1 %0 in scope" - -- id: "warning_invalid_locale_code" - msg: "unsupported locale code; supported locale codes are '%0'" diff --git a/test/diagnostics/Localization/Inputs/fr.strings b/test/diagnostics/Localization/Inputs/fr.strings new file mode 100644 index 00000000000..76c5be6ffc9 --- /dev/null +++ b/test/diagnostics/Localization/Inputs/fr.strings @@ -0,0 +1,27 @@ +/** + *===--- fr.strings - Localized diagnostic messages for French ------------===* + * + * This source file is part of the Swift.org open source project + * + * Copyright (c) 2014 - 2022 Apple Inc. and the Swift project authors + * Licensed under Apache License v2.0 with Runtime Library Exception + * + * See https://swift.org/LICENSE.txt for license information + * See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors + * + *===----------------------------------------------------------------------===* + * + * This file defines the diagnostic messages for the French language. + * Each diagnostic is described in the following format: + * "" = ""; + * + */ + +/* Different order from `en.strings` */ +"var_init_self_referential" = "variable utilisée dans sa propre valeur initiale"; + +"lex_unterminated_string" = "chaîne non terminée littérale"; + +"cannot_find_in_scope" = "impossible %select{de trouver|de trouver opérateur}1 %0 portée"; + +"warning_invalid_locale_code" = "code de paramètres régionaux non pris en charge; les codes pris en charge sont '%0'"; diff --git a/test/diagnostics/Localization/Inputs/fr.yaml b/test/diagnostics/Localization/Inputs/fr.yaml deleted file mode 100644 index e2a12327bf5..00000000000 --- a/test/diagnostics/Localization/Inputs/fr.yaml +++ /dev/null @@ -1,30 +0,0 @@ -#===--- fr.yaml - Localized diagnostic messages for French ---*- YAML -*-===# -# -# This source file is part of the Swift.org open source project -# -# Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors -# Licensed under Apache License v2.0 with Runtime Library Exception -# -# See https://swift.org/LICENSE.txt for license information -# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -# -#===----------------------------------------------------------------------===# -# -# This file defines the diagnostic messages for the French language. -# Each diagnostic is described in the following format: -# - id: "" -# msg: "" -# -#===----------------------------------------------------------------------===# - -- id: "lex_unterminated_string" - msg: "chaîne non terminée littérale" - -- id: "var_init_self_referential" - msg: "variable utilisée dans sa propre valeur initiale" - -- id: "cannot_find_in_scope" - msg: "impossible %select{de trouver|de trouver opérateur}1 %0 portée" - -- id: "warning_invalid_locale_code" - msg: "code de paramètres régionaux non pris en charge; les codes pris en charge sont '%0'" diff --git a/test/diagnostics/Localization/fr_debug_diagnostic_name.swift b/test/diagnostics/Localization/fr_debug_diagnostic_name.swift index 0046cbc9ff2..70e0cd7af54 100644 --- a/test/diagnostics/Localization/fr_debug_diagnostic_name.swift +++ b/test/diagnostics/Localization/fr_debug_diagnostic_name.swift @@ -1,6 +1,6 @@ // RUN: %empty-directory(%t) -// RUN: swift-serialize-diagnostics --input-file-path=%S/Inputs/fr.yaml --output-directory=%t/ -// RUN: swift-serialize-diagnostics --input-file-path=%S/Inputs/en.yaml --output-directory=%t/ +// RUN: swift-serialize-diagnostics --input-file-path=%S/Inputs/fr.strings --output-directory=%t/ +// RUN: swift-serialize-diagnostics --input-file-path=%S/Inputs/en.strings --output-directory=%t/ // RUN: not %target-swift-frontend -debug-diagnostic-names -localization-path %S/Inputs -locale fr -typecheck %s 2>&1 | %FileCheck %s --check-prefix=CHECK_NAMES _ = "HI! diff --git a/test/diagnostics/Localization/fr_localization.swift b/test/diagnostics/Localization/fr_localization.swift index 0c84ee6e56c..206fd65d579 100644 --- a/test/diagnostics/Localization/fr_localization.swift +++ b/test/diagnostics/Localization/fr_localization.swift @@ -1,6 +1,6 @@ // RUN: %empty-directory(%t) -// RUN: swift-serialize-diagnostics --input-file-path=%S/Inputs/fr.yaml --output-directory=%t/ -// RUN: swift-serialize-diagnostics --input-file-path=%S/Inputs/en.yaml --output-directory=%t/ +// RUN: swift-serialize-diagnostics --input-file-path=%S/Inputs/fr.strings --output-directory=%t/ +// RUN: swift-serialize-diagnostics --input-file-path=%S/Inputs/en.strings --output-directory=%t/ // RUN: %target-typecheck-verify-swift -localization-path %t -locale fr _ = "HI! diff --git a/test/diagnostics/Localization/no_localization_files_and_wrong_path.swift b/test/diagnostics/Localization/no_localization_files_and_wrong_path.swift index b17b8523a7e..5394acfc4c0 100644 --- a/test/diagnostics/Localization/no_localization_files_and_wrong_path.swift +++ b/test/diagnostics/Localization/no_localization_files_and_wrong_path.swift @@ -1,6 +1,6 @@ // RUN: %target-typecheck-verify-swift -localization-path /Nonexistent_path -locale en -// :0: warning: cannot find translations for 'en' at '/Nonexistent_path/en.yaml': no such file +// :0: warning: cannot find translations for 'en' at '/Nonexistent_path/en.strings': no such file // :0: warning: specified localization directory '/Nonexistent_path' does not exist, translation is disabled _ = "HI! diff --git a/tools/swift-serialize-diagnostics/swift-serialize-diagnostics.cpp b/tools/swift-serialize-diagnostics/swift-serialize-diagnostics.cpp index 6021df2b28a..65641537740 100644 --- a/tools/swift-serialize-diagnostics/swift-serialize-diagnostics.cpp +++ b/tools/swift-serialize-diagnostics/swift-serialize-diagnostics.cpp @@ -42,7 +42,7 @@ static llvm::cl::OptionCategory Category("swift-serialize-diagnostics Options"); static llvm::cl::opt InputFilePath("input-file-path", - llvm::cl::desc("Path to the YAML input file"), + llvm::cl::desc("Path to the YAML or `.strings` input file"), llvm::cl::cat(Category)); static llvm::cl::opt @@ -60,22 +60,44 @@ int main(int argc, char *argv[]) { "Swift Serialize Diagnostics Tool\n"); if (!llvm::sys::fs::exists(options::InputFilePath)) { - llvm::errs() << "YAML file not found\n"; + llvm::errs() << "diagnostics file not found\n"; return EXIT_FAILURE; } - YAMLLocalizationProducer yaml(options::InputFilePath); - auto localeCode = llvm::sys::path::filename(options::InputFilePath); llvm::SmallString<128> SerializedFilePath(options::OutputDirectory); llvm::sys::path::append(SerializedFilePath, localeCode); llvm::sys::path::replace_extension(SerializedFilePath, ".db"); SerializedLocalizationWriter Serializer; - yaml.forEachAvailable( - [&Serializer](swift::DiagID id, llvm::StringRef translation) { - Serializer.insert(id, translation); - }); + + if (llvm::sys::path::extension(options::InputFilePath) == ".yaml") { + YAMLLocalizationProducer yaml(options::InputFilePath); + + yaml.forEachAvailable( + [&Serializer](swift::DiagID id, llvm::StringRef translation) { + Serializer.insert(id, translation); + }); + + // Print out the diagnostics IDs that are available in YAML but not + // available in `.def` + if (!yaml.unknownIDs.empty()) { + llvm::errs() << "These diagnostic IDs are no longer available: '"; + llvm::interleave( + yaml.unknownIDs, [&](std::string id) { llvm::errs() << id; }, + [&] { llvm::errs() << ", "; }); + llvm::errs() << "'\n"; + } + } else { + assert(llvm::sys::path::extension(options::InputFilePath) == ".strings"); + + StringsLocalizationProducer strings(options::InputFilePath); + + strings.forEachAvailable( + [&Serializer](swift::DiagID id, llvm::StringRef translation) { + Serializer.insert(id, translation); + }); + } if (Serializer.emit(SerializedFilePath.str())) { llvm::errs() << "Cannot serialize diagnostic file " @@ -83,15 +105,5 @@ int main(int argc, char *argv[]) { return EXIT_FAILURE; } - // Print out the diagnostics IDs that are available in YAML but not available - // in `.def` - if (!yaml.unknownIDs.empty()) { - llvm::errs() << "These diagnostic IDs are no longer available: '"; - llvm::interleave( - yaml.unknownIDs, [&](std::string id) { llvm::errs() << id; }, - [&] { llvm::errs() << ", "; }); - llvm::errs() << "'\n"; - } - return EXIT_SUCCESS; } diff --git a/unittests/Localization/CMakeLists.txt b/unittests/Localization/CMakeLists.txt index 618d8927094..c2c5d961228 100644 --- a/unittests/Localization/CMakeLists.txt +++ b/unittests/Localization/CMakeLists.txt @@ -1,5 +1,5 @@ add_swift_unittest(swiftLocalizationTests - DefToYAMLConverterTests.cpp + DefToStringsConverterTests.cpp SerializationTests.cpp) target_link_libraries(swiftLocalizationTests diff --git a/unittests/Localization/DefToYAMLConverterTests.cpp b/unittests/Localization/DefToStringsConverterTests.cpp similarity index 85% rename from unittests/Localization/DefToYAMLConverterTests.cpp rename to unittests/Localization/DefToStringsConverterTests.cpp index 1683537cdb0..a2b74a311a5 100644 --- a/unittests/Localization/DefToYAMLConverterTests.cpp +++ b/unittests/Localization/DefToStringsConverterTests.cpp @@ -1,8 +1,8 @@ -//===--- DefToYAMLConverterTests.cpp -------------------------------------===// +//===--- DefToStringsConverterTests.cpp -----------------------------------===// // // This source file is part of the Swift.org open source project // -// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +// Copyright (c) 2014 - 2022 Apple Inc. and the Swift project authors // Licensed under Apache License v2.0 with Runtime Library Exception // // See https://swift.org/LICENSE.txt for license information @@ -49,22 +49,22 @@ TEST_F(LocalizationTest, MissingLocalizationFiles) { ASSERT_TRUE(llvm::sys::fs::exists(getDefaultLocalizationPath())); llvm::SmallString<128> EnglishLocalization(getDefaultLocalizationPath()); llvm::sys::path::append(EnglishLocalization, "en"); - llvm::sys::path::replace_extension(EnglishLocalization, ".yaml"); + llvm::sys::path::replace_extension(EnglishLocalization, ".strings"); ASSERT_TRUE(llvm::sys::fs::exists(EnglishLocalization)); llvm::sys::path::replace_extension(EnglishLocalization, ".db"); ASSERT_TRUE(llvm::sys::fs::exists(EnglishLocalization)); } TEST_F(LocalizationTest, ConverterTestMatchDiagnosticMessagesSequentially) { - YAMLLocalizationProducer yaml(YAMLPath); - yaml.forEachAvailable([](swift::DiagID id, llvm::StringRef translation) { + StringsLocalizationProducer strings(DiagsPath); + strings.forEachAvailable([](swift::DiagID id, llvm::StringRef translation) { llvm::StringRef msg = diagnosticMessages[static_cast(id)]; ASSERT_EQ(msg, translation); }); } TEST_F(LocalizationTest, ConverterTestMatchDiagnosticMessagesRandomly) { - YAMLLocalizationProducer yaml(YAMLPath); + StringsLocalizationProducer strings(DiagsPath); std::random_device rd; std::mt19937 gen(rd()); @@ -74,7 +74,7 @@ TEST_F(LocalizationTest, ConverterTestMatchDiagnosticMessagesRandomly) { unsigned randomNum = RandNumber(LocalDiagID::NumDiags); DiagID randomId = static_cast(randomNum); llvm::StringRef msg = diagnosticMessages[randomNum]; - llvm::StringRef translation = yaml.getMessageOr(randomId, ""); + llvm::StringRef translation = strings.getMessageOr(randomId, ""); ASSERT_EQ(msg, translation); } } diff --git a/unittests/Localization/LocalizationTest.h b/unittests/Localization/LocalizationTest.h index 6b78628c59a..7ca13314fcd 100644 --- a/unittests/Localization/LocalizationTest.h +++ b/unittests/Localization/LocalizationTest.h @@ -53,15 +53,15 @@ struct LocalizationTest : public ::testing::Test { llvm::SmallVector TempFiles; public: - std::string YAMLPath; + std::string DiagsPath; LocalizationTest() { - YAMLPath = std::string(createTemporaryFile("en", "yaml")); + DiagsPath = std::string(createTemporaryFile("en", "strings")); } void SetUp() override { - bool failed = convertDefIntoYAML(YAMLPath); - assert(!failed && "failed to generate a YAML file"); + bool failed = convertDefIntoStrings(DiagsPath); + assert(!failed && "failed to generate a `.strings` file"); } void TearDown() override { @@ -85,7 +85,7 @@ public: unsigned RandNumber(unsigned n) { return unsigned(rand()) % n; } protected: - static bool convertDefIntoYAML(std::string outputPath) { + static bool convertDefIntoStrings(std::string outputPath) { std::error_code error; llvm::raw_fd_ostream OS(outputPath, error, llvm::sys::fs::OF_None); if (OS.has_error() || error) @@ -95,7 +95,7 @@ protected: llvm::ArrayRef messages(diagnosticMessages, LocalDiagID::NumDiags); - DefToYAMLConverter converter(ids, messages); + DefToStringsConverter converter(ids, messages); converter.convert(OS); OS.flush(); diff --git a/unittests/Localization/SerializationTests.cpp b/unittests/Localization/SerializationTests.cpp index 5eea0419777..cf3048db6a9 100644 --- a/unittests/Localization/SerializationTests.cpp +++ b/unittests/Localization/SerializationTests.cpp @@ -24,8 +24,8 @@ using namespace swift::diag; using namespace swift::unittests; -TEST_F(LocalizationTest, TestYAMLSerialization) { - YAMLLocalizationProducer yaml(YAMLPath); +TEST_F(LocalizationTest, TestStringsSerialization) { + StringsLocalizationProducer strings(DiagsPath); auto dbFile = createTemporaryFile("en", "db"); @@ -33,9 +33,10 @@ TEST_F(LocalizationTest, TestYAMLSerialization) { { SerializedLocalizationWriter writer; - yaml.forEachAvailable([&writer](swift::DiagID id, llvm::StringRef translation) { - writer.insert(id, translation); - }); + strings.forEachAvailable( + [&writer](swift::DiagID id, llvm::StringRef translation) { + writer.insert(id, translation); + }); ASSERT_FALSE(writer.emit(dbFile)); } @@ -45,9 +46,10 @@ TEST_F(LocalizationTest, TestYAMLSerialization) { ASSERT_TRUE(dbContent); SerializedLocalizationProducer db(std::move(dbContent.get())); - yaml.forEachAvailable([&db](swift::DiagID id, llvm::StringRef translation) { - ASSERT_EQ(translation, db.getMessageOr(id, "")); - }); + strings.forEachAvailable( + [&db](swift::DiagID id, llvm::StringRef translation) { + ASSERT_EQ(translation, db.getMessageOr(id, "")); + }); } TEST_F(LocalizationTest, TestSerializationOfEmptyFile) { @@ -55,7 +57,7 @@ TEST_F(LocalizationTest, TestSerializationOfEmptyFile) { SerializedLocalizationWriter writer; ASSERT_FALSE(writer.emit(dbFile)); - YAMLLocalizationProducer yaml(YAMLPath); + StringsLocalizationProducer strings(DiagsPath); // Reading of the empty `db` file should always return default message. { @@ -63,7 +65,8 @@ TEST_F(LocalizationTest, TestSerializationOfEmptyFile) { ASSERT_TRUE(dbContent); SerializedLocalizationProducer db(std::move(dbContent.get())); - yaml.forEachAvailable([&db](swift::DiagID id, llvm::StringRef translation) { + strings.forEachAvailable([&db](swift::DiagID id, + llvm::StringRef translation) { ASSERT_EQ("<<>>", db.getMessageOr(id, "<<>>")); }); @@ -80,16 +83,17 @@ TEST_F(LocalizationTest, TestSerializationWithGaps) { includedMessages.flip(position); } - YAMLLocalizationProducer yaml(YAMLPath); + StringsLocalizationProducer strings(DiagsPath); auto dbFile = createTemporaryFile("en", "db"); { SerializedLocalizationWriter writer; - yaml.forEachAvailable([&](swift::DiagID id, llvm::StringRef translation) { - if (includedMessages.test((unsigned)id)) - writer.insert(id, translation); - }); + strings.forEachAvailable( + [&](swift::DiagID id, llvm::StringRef translation) { + if (includedMessages.test((unsigned)id)) + writer.insert(id, translation); + }); ASSERT_FALSE(writer.emit(dbFile)); } @@ -100,7 +104,8 @@ TEST_F(LocalizationTest, TestSerializationWithGaps) { ASSERT_TRUE(dbContent); SerializedLocalizationProducer db(std::move(dbContent.get())); - yaml.forEachAvailable([&](swift::DiagID id, llvm::StringRef translation) { + strings.forEachAvailable([&](swift::DiagID id, + llvm::StringRef translation) { auto position = (unsigned)id; std::string expectedMessage = includedMessages.test(position)