[stdlib] Shims for UBreakIterator and thread local storage.

Introduce shims for using UBreakIterators from ICU. Also introduce
shims for using thread local storage via pthreads.

We will be relying on ICU and UBreakIterators for grapheme
breaking. But, UBreakIterators are very expensive to create,
especially for the way we do grapheme breaking, which is relatively
stateless. Thus, we will stash one or more into thread local storage
and reset it as needed.

Note: Currently, pthread_key_t is hard coded for a single platform
(Darwin), but I have a static_assert alongside directions on how to
adapt it to any future platforms who differ in key type.

stdlib/public/SwiftShims/LibcShims.h

@@ -111,6 +111,29 @@ double _swift_stdlib_squareRoot(double _self) {
   return __builtin_sqrt(_self);
 }
 
+// TLS - thread local storage
+
+// FIXME: multi-platform
+typedef unsigned long __swift_pthread_key_t;
+
+SWIFT_RUNTIME_STDLIB_INTERFACE
+int _swift_stdlib_pthread_key_create(
+  __swift_pthread_key_t * _Nonnull key, void
+  (* _Nullable destructor)(void * _Nullable )
+);
+
+SWIFT_RUNTIME_STDLIB_INTERFACE
+void * _Nullable _swift_stdlib_pthread_getspecific(__swift_pthread_key_t key);
+
+SWIFT_RUNTIME_STDLIB_INTERFACE
+int _swift_stdlib_pthread_setspecific(
+  __swift_pthread_key_t key, const void * _Nullable value
+);
+
+SWIFT_RUNTIME_STDLIB_INTERFACE
+__swift_pthread_key_t _swift_stdlib_PTHREAD_KEYS_MAX(void);
+
+
 // TODO: Remove horrible workaround when importer does Float80 <-> long double.
 #if (defined __i386__ || defined __x86_64__) && !defined _MSC_VER
 static inline SWIFT_ALWAYS_INLINE

stdlib/public/SwiftShims/UnicodeShims.h

@@ -109,6 +109,218 @@ __swift_int32_t _swift_stdlib_unicode_strToLower(
   __swift_uint16_t *Destination, __swift_int32_t DestinationCapacity,
   const __swift_uint16_t *Source, __swift_int32_t SourceLength);
 
+typedef enum __swift_stdlib_UErrorCode {
+  __swift_stdlib_U_USING_FALLBACK_WARNING = -128,
+  __swift_stdlib_U_ERROR_WARNING_START = -128,
+  __swift_stdlib_U_USING_DEFAULT_WARNING = -127,
+  __swift_stdlib_U_SAFECLONE_ALLOCATED_WARNING = -126,
+  __swift_stdlib_U_STATE_OLD_WARNING = -125,
+  __swift_stdlib_U_STRING_NOT_TERMINATED_WARNING = -124,
+  __swift_stdlib_U_SORT_KEY_TOO_SHORT_WARNING = -123,
+  __swift_stdlib_U_AMBIGUOUS_ALIAS_WARNING = -122,
+  __swift_stdlib_U_DIFFERENT_UCA_VERSION = -121,
+  __swift_stdlib_U_PLUGIN_CHANGED_LEVEL_WARNING = -120,
+  __swift_stdlib_U_ERROR_WARNING_LIMIT,
+  __swift_stdlib_U_ZERO_ERROR = 0,
+  __swift_stdlib_U_ILLEGAL_ARGUMENT_ERROR = 1,
+  __swift_stdlib_U_MISSING_RESOURCE_ERROR = 2,
+  __swift_stdlib_U_INVALID_FORMAT_ERROR = 3,
+  __swift_stdlib_U_FILE_ACCESS_ERROR = 4,
+  __swift_stdlib_U_INTERNAL_PROGRAM_ERROR = 5,
+  __swift_stdlib_U_MESSAGE_PARSE_ERROR = 6,
+  __swift_stdlib_U_MEMORY_ALLOCATION_ERROR = 7,
+  __swift_stdlib_U_INDEX_OUTOFBOUNDS_ERROR = 8,
+  __swift_stdlib_U_PARSE_ERROR = 9,
+  __swift_stdlib_U_INVALID_CHAR_FOUND = 10,
+  __swift_stdlib_U_TRUNCATED_CHAR_FOUND = 11,
+  __swift_stdlib_U_ILLEGAL_CHAR_FOUND = 12,
+  __swift_stdlib_U_INVALID_TABLE_FORMAT = 13,
+  __swift_stdlib_U_INVALID_TABLE_FILE = 14,
+  __swift_stdlib_U_BUFFER_OVERFLOW_ERROR = 15,
+  __swift_stdlib_U_UNSUPPORTED_ERROR = 16,
+  __swift_stdlib_U_RESOURCE_TYPE_MISMATCH = 17,
+  __swift_stdlib_U_ILLEGAL_ESCAPE_SEQUENCE = 18,
+  __swift_stdlib_U_UNSUPPORTED_ESCAPE_SEQUENCE = 19,
+  __swift_stdlib_U_NO_SPACE_AVAILABLE = 20,
+  __swift_stdlib_U_CE_NOT_FOUND_ERROR = 21,
+  __swift_stdlib_U_PRIMARY_TOO_LONG_ERROR = 22,
+  __swift_stdlib_U_STATE_TOO_OLD_ERROR = 23,
+  __swift_stdlib_U_TOO_MANY_ALIASES_ERROR = 24,
+  __swift_stdlib_U_ENUM_OUT_OF_SYNC_ERROR = 25,
+  __swift_stdlib_U_INVARIANT_CONVERSION_ERROR = 26,
+  __swift_stdlib_U_INVALID_STATE_ERROR = 27,
+  __swift_stdlib_U_COLLATOR_VERSION_MISMATCH = 28,
+  __swift_stdlib_U_USELESS_COLLATOR_ERROR = 29,
+  __swift_stdlib_U_NO_WRITE_PERMISSION = 30,
+  __swift_stdlib_U_STANDARD_ERROR_LIMIT,
+  __swift_stdlib_U_BAD_VARIABLE_DEFINITION = 0x10000,
+  __swift_stdlib_U_PARSE_ERROR_START = 0x10000,
+  __swift_stdlib_U_MALFORMED_RULE,
+  __swift_stdlib_U_MALFORMED_SET,
+  __swift_stdlib_U_MALFORMED_SYMBOL_REFERENCE,
+  __swift_stdlib_U_MALFORMED_UNICODE_ESCAPE,
+  __swift_stdlib_U_MALFORMED_VARIABLE_DEFINITION,
+  __swift_stdlib_U_MALFORMED_VARIABLE_REFERENCE,
+  __swift_stdlib_U_MISMATCHED_SEGMENT_DELIMITERS,
+  __swift_stdlib_U_MISPLACED_ANCHOR_START,
+  __swift_stdlib_U_MISPLACED_CURSOR_OFFSET,
+  __swift_stdlib_U_MISPLACED_QUANTIFIER,
+  __swift_stdlib_U_MISSING_OPERATOR,
+  __swift_stdlib_U_MISSING_SEGMENT_CLOSE,
+  __swift_stdlib_U_MULTIPLE_ANTE_CONTEXTS,
+  __swift_stdlib_U_MULTIPLE_CURSORS,
+  __swift_stdlib_U_MULTIPLE_POST_CONTEXTS,
+  __swift_stdlib_U_TRAILING_BACKSLASH,
+  __swift_stdlib_U_UNDEFINED_SEGMENT_REFERENCE,
+  __swift_stdlib_U_UNDEFINED_VARIABLE,
+  __swift_stdlib_U_UNQUOTED_SPECIAL,
+  __swift_stdlib_U_UNTERMINATED_QUOTE,
+  __swift_stdlib_U_RULE_MASK_ERROR,
+  __swift_stdlib_U_MISPLACED_COMPOUND_FILTER,
+  __swift_stdlib_U_MULTIPLE_COMPOUND_FILTERS,
+  __swift_stdlib_U_INVALID_RBT_SYNTAX,
+  __swift_stdlib_U_INVALID_PROPERTY_PATTERN,
+  __swift_stdlib_U_MALFORMED_PRAGMA,
+  __swift_stdlib_U_UNCLOSED_SEGMENT,
+  __swift_stdlib_U_ILLEGAL_CHAR_IN_SEGMENT,
+  __swift_stdlib_U_VARIABLE_RANGE_EXHAUSTED,
+  __swift_stdlib_U_VARIABLE_RANGE_OVERLAP,
+  __swift_stdlib_U_ILLEGAL_CHARACTER,
+  __swift_stdlib_U_INTERNAL_TRANSLITERATOR_ERROR,
+  __swift_stdlib_U_INVALID_ID,
+  __swift_stdlib_U_INVALID_FUNCTION,
+  __swift_stdlib_U_PARSE_ERROR_LIMIT,
+  __swift_stdlib_U_UNEXPECTED_TOKEN = 0x10100,
+  __swift_stdlib_U_FMT_PARSE_ERROR_START = 0x10100,
+  __swift_stdlib_U_MULTIPLE_DECIMAL_SEPARATORS,
+  __swift_stdlib_U_MULTIPLE_DECIMAL_SEPERATORS =
+      __swift_stdlib_U_MULTIPLE_DECIMAL_SEPARATORS,
+  __swift_stdlib_U_MULTIPLE_EXPONENTIAL_SYMBOLS,
+  __swift_stdlib_U_MALFORMED_EXPONENTIAL_PATTERN,
+  __swift_stdlib_U_MULTIPLE_PERCENT_SYMBOLS,
+  __swift_stdlib_U_MULTIPLE_PERMILL_SYMBOLS,
+  __swift_stdlib_U_MULTIPLE_PAD_SPECIFIERS,
+  __swift_stdlib_U_PATTERN_SYNTAX_ERROR,
+  __swift_stdlib_U_ILLEGAL_PAD_POSITION,
+  __swift_stdlib_U_UNMATCHED_BRACES,
+  __swift_stdlib_U_UNSUPPORTED_PROPERTY,
+  __swift_stdlib_U_UNSUPPORTED_ATTRIBUTE,
+  __swift_stdlib_U_ARGUMENT_TYPE_MISMATCH,
+  __swift_stdlib_U_DUPLICATE_KEYWORD,
+  __swift_stdlib_U_UNDEFINED_KEYWORD,
+  __swift_stdlib_U_DEFAULT_KEYWORD_MISSING,
+  __swift_stdlib_U_DECIMAL_NUMBER_SYNTAX_ERROR,
+  __swift_stdlib_U_FORMAT_INEXACT_ERROR,
+  __swift_stdlib_U_FMT_PARSE_ERROR_LIMIT,
+  __swift_stdlib_U_BRK_INTERNAL_ERROR = 0x10200,
+  __swift_stdlib_U_BRK_ERROR_START = 0x10200,
+  __swift_stdlib_U_BRK_HEX_DIGITS_EXPECTED,
+  __swift_stdlib_U_BRK_SEMICOLON_EXPECTED,
+  __swift_stdlib_U_BRK_RULE_SYNTAX,
+  __swift_stdlib_U_BRK_UNCLOSED_SET,
+  __swift_stdlib_U_BRK_ASSIGN_ERROR,
+  __swift_stdlib_U_BRK_VARIABLE_REDFINITION,
+  __swift_stdlib_U_BRK_MISMATCHED_PAREN,
+  __swift_stdlib_U_BRK_NEW_LINE_IN_QUOTED_STRING,
+  __swift_stdlib_U_BRK_UNDEFINED_VARIABLE,
+  __swift_stdlib_U_BRK_INIT_ERROR,
+  __swift_stdlib_U_BRK_RULE_EMPTY_SET,
+  __swift_stdlib_U_BRK_UNRECOGNIZED_OPTION,
+  __swift_stdlib_U_BRK_MALFORMED_RULE_TAG,
+  __swift_stdlib_U_BRK_ERROR_LIMIT,
+  __swift_stdlib_U_REGEX_INTERNAL_ERROR = 0x10300,
+  __swift_stdlib_U_REGEX_ERROR_START = 0x10300,
+  __swift_stdlib_U_REGEX_RULE_SYNTAX,
+  __swift_stdlib_U_REGEX_INVALID_STATE,
+  __swift_stdlib_U_REGEX_BAD_ESCAPE_SEQUENCE,
+  __swift_stdlib_U_REGEX_PROPERTY_SYNTAX,
+  __swift_stdlib_U_REGEX_UNIMPLEMENTED,
+  __swift_stdlib_U_REGEX_MISMATCHED_PAREN,
+  __swift_stdlib_U_REGEX_NUMBER_TOO_BIG,
+  __swift_stdlib_U_REGEX_BAD_INTERVAL,
+  __swift_stdlib_U_REGEX_MAX_LT_MIN,
+  __swift_stdlib_U_REGEX_INVALID_BACK_REF,
+  __swift_stdlib_U_REGEX_INVALID_FLAG,
+  __swift_stdlib_U_REGEX_LOOK_BEHIND_LIMIT,
+  __swift_stdlib_U_REGEX_SET_CONTAINS_STRING,
+#ifndef __swift_stdlib_U_HIDE_DEPRECATED_API
+  __swift_stdlib_U_REGEX_OCTAL_TOO_BIG,
+#endif
+  __swift_stdlib_U_REGEX_MISSING_CLOSE_BRACKET =
+      __swift_stdlib_U_REGEX_SET_CONTAINS_STRING + 2,
+  __swift_stdlib_U_REGEX_INVALID_RANGE,
+  __swift_stdlib_U_REGEX_STACK_OVERFLOW,
+  __swift_stdlib_U_REGEX_TIME_OUT,
+  __swift_stdlib_U_REGEX_STOPPED_BY_CALLER,
+#ifndef __swift_stdlib_U_HIDE_DRAFT_API
+  __swift_stdlib_U_REGEX_PATTERN_TOO_BIG,
+  __swift_stdlib_U_REGEX_INVALID_CAPTURE_GROUP_NAME,
+#endif
+  __swift_stdlib_U_REGEX_ERROR_LIMIT =
+      __swift_stdlib_U_REGEX_STOPPED_BY_CALLER + 3,
+  __swift_stdlib_U_IDNA_PROHIBITED_ERROR = 0x10400,
+  __swift_stdlib_U_IDNA_ERROR_START = 0x10400,
+  __swift_stdlib_U_IDNA_UNASSIGNED_ERROR,
+  __swift_stdlib_U_IDNA_CHECK_BIDI_ERROR,
+  __swift_stdlib_U_IDNA_STD3_ASCII_RULES_ERROR,
+  __swift_stdlib_U_IDNA_ACE_PREFIX_ERROR,
+  __swift_stdlib_U_IDNA_VERIFICATION_ERROR,
+  __swift_stdlib_U_IDNA_LABEL_TOO_LONG_ERROR,
+  __swift_stdlib_U_IDNA_ZERO_LENGTH_LABEL_ERROR,
+  __swift_stdlib_U_IDNA_DOMAIN_NAME_TOO_LONG_ERROR,
+  __swift_stdlib_U_IDNA_ERROR_LIMIT,
+  __swift_stdlib_U_STRINGPREP_PROHIBITED_ERROR =
+      __swift_stdlib_U_IDNA_PROHIBITED_ERROR,
+  __swift_stdlib_U_STRINGPREP_UNASSIGNED_ERROR =
+      __swift_stdlib_U_IDNA_UNASSIGNED_ERROR,
+  __swift_stdlib_U_STRINGPREP_CHECK_BIDI_ERROR =
+      __swift_stdlib_U_IDNA_CHECK_BIDI_ERROR,
+  __swift_stdlib_U_PLUGIN_ERROR_START = 0x10500,
+  __swift_stdlib_U_PLUGIN_TOO_HIGH = 0x10500,
+  __swift_stdlib_U_PLUGIN_DIDNT_SET_LEVEL,
+  __swift_stdlib_U_PLUGIN_ERROR_LIMIT,
+  __swift_stdlib_U_ERROR_LIMIT = __swift_stdlib_U_PLUGIN_ERROR_LIMIT
+} __swift_stdlib_UErrorCode;
+
+typedef enum __swift_stdlib_UBreakIteratorType {
+  __swift_stdlib_UBRK_CHARACTER = 0,
+  __swift_stdlib_UBRK_WORD = 1,
+  __swift_stdlib_UBRK_LINE = 2,
+  __swift_stdlib_UBRK_SENTENCE = 3,
+#ifndef U_HIDE_DEPRECATED_API
+  __swift_stdlib_UBRK_TITLE = 4,
+#endif
+  __swift_stdlib_UBRK_COUNT = 5
+} __swift_stdlib_UBreakIteratorType;
+
+typedef struct __swift_stdlib_UBreakIterator __swift_stdlib_UBreakIterator;
+typedef __swift_uint16_t __swift_stdlib_UChar;
+
+SWIFT_RUNTIME_STDLIB_INTERFACE
+void __swift_stdlib_ubrk_close(__swift_stdlib_UBreakIterator *bi);
+
+SWIFT_RUNTIME_STDLIB_INTERFACE
+__swift_stdlib_UBreakIterator *
+__swift_stdlib_ubrk_open(__swift_stdlib_UBreakIteratorType type,
+                         const char *_Null_unspecified locale,
+                         const __swift_stdlib_UChar *_Null_unspecified text,
+                         __swift_int32_t textLength,
+                         __swift_stdlib_UErrorCode *status);
+
+SWIFT_RUNTIME_STDLIB_INTERFACE
+void __swift_stdlib_ubrk_setText(__swift_stdlib_UBreakIterator *bi,
+                                 const __swift_stdlib_UChar *text,
+                                 __swift_int32_t textLength,
+                                 __swift_stdlib_UErrorCode *status);
+
+SWIFT_RUNTIME_STDLIB_INTERFACE
+__swift_int32_t __swift_stdlib_ubrk_preceding(__swift_stdlib_UBreakIterator *bi,
+                                              __swift_int32_t offset);
+
+SWIFT_RUNTIME_STDLIB_INTERFACE
+__swift_int32_t __swift_stdlib_ubrk_following(__swift_stdlib_UBreakIterator *bi,
+                                              __swift_int32_t offset);
+
 #ifdef __cplusplus
 }} // extern "C", namespace swift
 #endif

stdlib/public/core/CMakeLists.txt

@@ -176,6 +176,7 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
   list(APPEND swift_core_link_flags "-all_load")
   list(APPEND swift_core_framework_depends Foundation)
   list(APPEND swift_core_framework_depends CoreFoundation)
+  list(APPEND swift_core_private_link_libraries icucore)
 else()
   # With the GNU linker the equivalent of -all_load is to tell the linker
   # --whole-archive before the archive and --no-whole-archive after (without

stdlib/public/stubs/CMakeLists.txt

@@ -31,6 +31,12 @@ set(ICU_UC_LIBRARY "")
 
 set(swift_stubs_c_compile_flags ${SWIFT_RUNTIME_CORE_CXX_FLAGS})
 list(APPEND swift_stubs_c_compile_flags -DswiftCore_EXPORTS)
+if(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
+  # Our shims need ICU headers. For now, use the ones inside corelibs-foundation
+  #
+  # TODO: Get them from the SDK instead
+  list(APPEND swift_stubs_c_compile_flags -I${CMAKE_CURRENT_LIST_DIR}/../../../../swift-corelibs-foundation/bootstrap/x86_64-apple-darwin/usr/local/include/)
+endif()
 
 add_swift_library(swiftStdlibStubs OBJECT_LIBRARY TARGET_LIBRARY
   ${swift_stubs_sources}

stdlib/public/stubs/LibcShims.cpp

@@ -96,6 +96,40 @@ int swift::_swift_stdlib_close(int fd) {
 #endif
 }
 
+// Guard compilation on the typedef for __swift_pthread_key_t in LibcShims.h
+// being identical to the platform's pthread_key_t
+static_assert(std::is_same<__swift_pthread_key_t, pthread_key_t>::value,
+              "This platform's pthread_key_t differs. If you hit this assert, "
+              "fix __swift_pthread_key_t's typedef in LibcShims.h by adding an "
+              "#if guard and definition for your platform");
+
+SWIFT_RUNTIME_STDLIB_INTERFACE
+int swift::_swift_stdlib_pthread_key_create(
+  __swift_pthread_key_t * _Nonnull key,
+  void (* _Nullable destructor)(void *)
+) {
+  return pthread_key_create(key, destructor);
+}
+
+SWIFT_RUNTIME_STDLIB_INTERFACE
+void * _Nullable swift::_swift_stdlib_pthread_getspecific(
+  __swift_pthread_key_t key
+) {
+  return pthread_getspecific(key);
+}
+
+SWIFT_RUNTIME_STDLIB_INTERFACE
+int swift::_swift_stdlib_pthread_setspecific(
+  __swift_pthread_key_t key, const void * _Nullable value
+) {
+  return pthread_setspecific(key, value);
+}
+
+SWIFT_RUNTIME_STDLIB_INTERFACE
+__swift_pthread_key_t _swift_stdlib_PTHREAD_KEYS_MAX(void) {
+  return PTHREAD_KEYS_MAX;
+}
+
 #if defined(__APPLE__)
 #include <malloc/malloc.h>
 SWIFT_RUNTIME_STDLIB_INTERFACE

stdlib/public/stubs/UnicodeNormalization.cpp

@@ -22,6 +22,7 @@
 #include <algorithm>
 #include <mutex>
 #include <assert.h>
+#endif
 
 #pragma clang diagnostic push
 #pragma clang diagnostic ignored "-Wdocumentation"
@@ -30,11 +31,13 @@
 #include <unicode/ucol.h>
 #include <unicode/ucoleitr.h>
 #include <unicode/uiter.h>
+#include <unicode/ubrk.h>
 
 #pragma clang diagnostic pop
 
 #include "../SwiftShims/UnicodeShims.h"
 
+#if !defined(__APPLE__)
 static const UCollator *MakeRootCollator() {
   UErrorCode ErrorCode = U_ZERO_ERROR;
   UCollator *root = ucol_open("", &ErrorCode);
@@ -291,3 +294,42 @@ swift::_swift_stdlib_unicode_strToLower(uint16_t *Destination,
 swift::Lazy<ASCIICollation> ASCIICollation::theTable;
 #endif
 
+namespace {
+template <typename T, typename U> T *ptr_cast(U *p) {
+  return static_cast<T *>(static_cast<void *>(p));
+}
+template <typename T, typename U> const T *ptr_cast(const U *p) {
+  return static_cast<const T *>(static_cast<const void *>(p));
+}
+}
+
+void swift::__swift_stdlib_ubrk_close(
+    swift::__swift_stdlib_UBreakIterator *bi) {
+  ubrk_close(ptr_cast<UBreakIterator>(bi));
+}
+
+swift::__swift_stdlib_UBreakIterator *swift::__swift_stdlib_ubrk_open(
+    swift::__swift_stdlib_UBreakIteratorType type, const char *locale,
+    const UChar *text, int32_t textLength, __swift_stdlib_UErrorCode *status) {
+  return ptr_cast<swift::__swift_stdlib_UBreakIterator>(
+      ubrk_open(static_cast<UBreakIteratorType>(type), locale, text, textLength,
+                ptr_cast<UErrorCode>(status)));
+}
+
+int32_t
+swift::__swift_stdlib_ubrk_preceding(swift::__swift_stdlib_UBreakIterator *bi,
+                                     int32_t offset) {
+  return ubrk_preceding(ptr_cast<UBreakIterator>(bi), offset);
+}
+
+int32_t
+swift::__swift_stdlib_ubrk_following(swift::__swift_stdlib_UBreakIterator *bi,
+                                     int32_t offset) {
+  return ubrk_following(ptr_cast<UBreakIterator>(bi), offset);
+}
+void swift::__swift_stdlib_ubrk_setText(
+    swift::__swift_stdlib_UBreakIterator *bi, const __swift_stdlib_UChar *text,
+    __swift_int32_t textLength, __swift_stdlib_UErrorCode *status) {
+  return ubrk_setText(ptr_cast<UBreakIterator>(bi), ptr_cast<UChar>(text),
+                      textLength, ptr_cast<UErrorCode>(status));
+}