Support in-place value metadata initialization in the runtime.

include/swift/ABI/Metadata.h

@@ -3364,6 +3364,14 @@ public:
     return TypeContextDescriptorFlags(this->Flags.getKindSpecificFlags());
   }
 
+  /// Does this type have non-trivial "in place" metadata initialization?
+  ///
+  /// The type of the initialization-control structure differs by subclass,
+  /// so it doesn't appear here.
+  bool hasInPlaceMetadataInitialization() const {
+    return getTypeContextDescriptorFlags().hasInPlaceMetadataInitialization();
+  }
+
   const TargetTypeGenericContextDescriptorHeader<Runtime> &
     getFullGenericContextHeader() const;
 
@@ -3709,10 +3717,49 @@ public:
 
 using ClassDescriptor = TargetClassDescriptor<InProcess>;
 
+/// The cache structure for non-trivial initialization of singleton value
+/// metadata.
+template <typename Runtime>
+struct TargetInPlaceValueMetadataCache {
+  /// The metadata pointer.  Clients can do dependency-ordered loads
+  /// from this, and if they see a non-zero value, it's a Complete
+  /// metadata.
+  std::atomic<TargetMetadataPointer<Runtime, TargetMetadata>> Metadata;
+
+  /// The private cache data.
+  std::atomic<TargetPointer<Runtime, void>> Private;
+};
+using InPlaceValueMetadataCache =
+  TargetInPlaceValueMetadataCache<InProcess>;
+
+/// The control structure for performing non-trivial initialization of
+/// singleton value metadata, which is required when e.g. a non-generic
+/// value type has a resilient component type.
+template <typename Runtime>
+struct TargetInPlaceValueMetadataInitialization {
+  /// The initialization cache.  Out-of-line because mutable.
+  TargetRelativeDirectPointer<Runtime,
+                              TargetInPlaceValueMetadataCache<Runtime>>
+    InitializationCache;
+
+  /// The incomplete metadata.
+  TargetRelativeDirectPointer<Runtime, TargetMetadata<Runtime>>
+    IncompleteMetadata;
+
+  /// The completion function.  The pattern will always be null.
+  TargetRelativeDirectPointer<Runtime, MetadataCompleter>
+    CompletionFunction;
+};
+
 template <typename Runtime>
 class TargetValueTypeDescriptor
     : public TargetTypeContextDescriptor<Runtime> {
 public:
+  using InPlaceMetadataInitialization =
+    TargetInPlaceValueMetadataInitialization<Runtime>;
+
+  const InPlaceMetadataInitialization &getInPlaceMetadataInitialization() const;
+
   static bool classof(const TargetContextDescriptor<Runtime> *cd) {
     return cd->getKind() == ContextDescriptorKind::Struct ||
            cd->getKind() == ContextDescriptorKind::Enum;
@@ -3724,16 +3771,30 @@ template <typename Runtime>
 class TargetStructDescriptor final
     : public TargetValueTypeDescriptor<Runtime>,
       public TrailingGenericContextObjects<TargetStructDescriptor<Runtime>,
-                            TargetTypeGenericContextDescriptorHeader> {
+                            TargetTypeGenericContextDescriptorHeader,
+                            TargetInPlaceValueMetadataInitialization<Runtime>> {
+public:
+  using InPlaceMetadataInitialization =
+    TargetInPlaceValueMetadataInitialization<Runtime>;
+
 private:
   using TrailingGenericContextObjects =
     TrailingGenericContextObjects<TargetStructDescriptor<Runtime>,
-                                  TargetTypeGenericContextDescriptorHeader>;
+                                  TargetTypeGenericContextDescriptorHeader,
+                                  InPlaceMetadataInitialization>;
 
   using TrailingObjects =
     typename TrailingGenericContextObjects::TrailingObjects;
   friend TrailingObjects;
 
+  template<typename T>
+  using OverloadToken = typename TrailingObjects::template OverloadToken<T>;
+
+  using TrailingGenericContextObjects::numTrailingObjects;
+  size_t numTrailingObjects(OverloadToken<InPlaceMetadataInitialization>) const{
+    return this->hasInPlaceMetadataInitialization() ? 1 : 0;
+  }
+
 public:
   using TrailingGenericContextObjects::getGenericContext;
   using TrailingGenericContextObjects::getGenericContextHeader;
@@ -3752,6 +3813,11 @@ public:
   /// its stored properties.
   bool hasFieldOffsetVector() const { return FieldOffsetVectorOffset != 0; }
 
+  const InPlaceMetadataInitialization &getInPlaceMetadataInitialization() const{
+    assert(this->hasInPlaceMetadataInitialization());
+    return *this->template getTrailingObjects<InPlaceMetadataInitialization>();
+  }
+
   static constexpr int32_t getGenericArgumentOffset() {
     return TargetStructMetadata<Runtime>::getGenericArgumentOffset();
   }
@@ -3767,16 +3833,30 @@ template <typename Runtime>
 class TargetEnumDescriptor final
     : public TargetValueTypeDescriptor<Runtime>,
       public TrailingGenericContextObjects<TargetEnumDescriptor<Runtime>,
-                                     TargetTypeGenericContextDescriptorHeader> {
+                            TargetTypeGenericContextDescriptorHeader,
+                            TargetInPlaceValueMetadataInitialization<Runtime>> {
+public:
+  using InPlaceMetadataInitialization =
+    TargetInPlaceValueMetadataInitialization<Runtime>;
+
 private:
   using TrailingGenericContextObjects =
     TrailingGenericContextObjects<TargetEnumDescriptor<Runtime>,
-                                  TargetTypeGenericContextDescriptorHeader>;
+                                  TargetTypeGenericContextDescriptorHeader,
+                                  InPlaceMetadataInitialization>;
 
   using TrailingObjects =
     typename TrailingGenericContextObjects::TrailingObjects;
   friend TrailingObjects;
 
+  template<typename T>
+  using OverloadToken = typename TrailingObjects::template OverloadToken<T>;
+
+  using TrailingGenericContextObjects::numTrailingObjects;
+  size_t numTrailingObjects(OverloadToken<InPlaceMetadataInitialization>) const{
+    return this->hasInPlaceMetadataInitialization() ? 1 : 0;
+  }
+
 public:
   using TrailingGenericContextObjects::getGenericContext;
   using TrailingGenericContextObjects::getGenericContextHeader;
@@ -3813,6 +3893,11 @@ public:
     return TargetEnumMetadata<Runtime>::getGenericArgumentOffset();
   }
 
+  const InPlaceMetadataInitialization &getInPlaceMetadataInitialization() const{
+    assert(this->hasInPlaceMetadataInitialization());
+    return *this->template getTrailingObjects<InPlaceMetadataInitialization>();
+  }
+
   static bool classof(const TargetContextDescriptor<Runtime> *cd) {
     return cd->getKind() == ContextDescriptorKind::Enum;
   }
@@ -3901,6 +3986,21 @@ TargetTypeContextDescriptor<Runtime>::getGenericParams() const {
   }
 }
 
+template<typename Runtime>
+inline const TargetInPlaceValueMetadataInitialization<Runtime> &
+TargetValueTypeDescriptor<Runtime>::getInPlaceMetadataInitialization() const {
+  switch (this->getKind()) {
+  case ContextDescriptorKind::Enum:
+    return llvm::cast<TargetEnumDescriptor<Runtime>>(this)
+        ->getInPlaceMetadataInitialization();
+  case ContextDescriptorKind::Struct:
+    return llvm::cast<TargetStructDescriptor<Runtime>>(this)
+        ->getInPlaceMetadataInitialization();
+  default:
+    swift_runtime_unreachable("Not a value type descriptor.");
+  }
+}
+
 } // end namespace swift
 
 #pragma clang diagnostic pop

include/swift/ABI/MetadataValues.h

@@ -1202,6 +1202,14 @@ class TypeContextDescriptorFlags : public FlagSet<uint16_t> {
     /// declarations associated with the same declaration.
     IsSynthesizedRelatedEntity = 3,
 
+    /// Set if the type requires non-trivial but non-generic metadata
+    /// initialization.  It may or may not be truly "in place" depending
+    /// on the kind of metadata.
+    ///
+    /// Currently only meaningful for value descriptors, but will be
+    /// extended to class descriptors.
+    HasInPlaceMetadataInitialization = 4,
+
     /// Set if the context descriptor is includes metadata for dynamically
     /// constructing a class's vtables at metadata instantiation time.
     ///
@@ -1237,6 +1245,10 @@ public:
                                 isSynthesizedRelatedEntity,
                                 setIsSynthesizedRelatedEntity)
 
+  FLAGSET_DEFINE_FLAG_ACCESSORS(HasInPlaceMetadataInitialization,
+                                hasInPlaceMetadataInitialization,
+                                setHasInPlaceMetadataInitialization)
+
   FLAGSET_DEFINE_FLAG_ACCESSORS(Class_HasVTable,
                                 class_hasVTable,
                                 class_setHasVTable)

include/swift/Remote/MetadataReader.h

@@ -1050,8 +1050,10 @@ private:
                            sizeof(flags)))
       return nullptr;
     
+    TypeContextDescriptorFlags typeFlags(flags.getKindSpecificFlags());
     unsigned baseSize = 0;
     unsigned genericHeaderSize = sizeof(GenericContextDescriptorHeader);
+    unsigned inPlaceInitSize = 0;
     bool hasVTable = false;
     switch (auto kind = flags.getKind()) {
     case ContextDescriptorKind::Module:
@@ -1067,16 +1069,23 @@ private:
     case ContextDescriptorKind::Class:
       baseSize = sizeof(TargetClassDescriptor<Runtime>);
       genericHeaderSize = sizeof(TypeGenericContextDescriptorHeader);
-      hasVTable = TypeContextDescriptorFlags(flags.getKindSpecificFlags())
+      hasVTable = typeFlags.class_hasVTable();
-                    .class_hasVTable();
       break;
     case ContextDescriptorKind::Enum:
       baseSize = sizeof(TargetEnumDescriptor<Runtime>);
       genericHeaderSize = sizeof(TypeGenericContextDescriptorHeader);
+      if (typeFlags.hasInPlaceMetadataInitialization()) {
+        inPlaceInitSize =
+          sizeof(TargetInPlaceValueMetadataInitialization<Runtime>);
+      }
       break;
     case ContextDescriptorKind::Struct:
       baseSize = sizeof(TargetStructDescriptor<Runtime>);
       genericHeaderSize = sizeof(TypeGenericContextDescriptorHeader);
+      if (typeFlags.hasInPlaceMetadataInitialization()) {
+        inPlaceInitSize =
+          sizeof(TargetInPlaceValueMetadataInitialization<Runtime>);
+      }
       break;
     case ContextDescriptorKind::Protocol:
       baseSize = sizeof(TargetProtocolDescriptorRef<Runtime>);
@@ -1122,7 +1131,7 @@ private:
         + header.VTableSize * sizeof(TargetMethodDescriptor<Runtime>);
     }
     
-    unsigned size = baseSize + genericsSize + vtableSize;
+    unsigned size = baseSize + genericsSize + vtableSize + inPlaceInitSize;
     auto buffer = (uint8_t *)malloc(size);
     if (!Reader->readBytes(RemoteAddress(address), buffer, size)) {
       free(buffer);

include/swift/Runtime/Metadata.h

@@ -335,6 +335,13 @@ ClassMetadataBounds getResilientMetadataBounds(
                                            const ClassDescriptor *descriptor);
 int32_t getResilientImmediateMembersOffset(const ClassDescriptor *descriptor);
 
+/// \brief Fetch a uniqued metadata object for a nominal type which requires
+/// in-place metadata initialization.
+SWIFT_RUNTIME_EXPORT SWIFT_CC(swift)
+MetadataResponse
+swift_getInPlaceMetadata(MetadataRequest request,
+                         const TypeContextDescriptor *description);
+
 /// \brief Fetch a uniqued metadata object for a generic nominal type.
 SWIFT_RUNTIME_EXPORT SWIFT_CC(swift)
 MetadataResponse

include/swift/Runtime/RuntimeFunctions.def

@@ -653,6 +653,13 @@ FUNCTION(GetForeignWitnessTable, swift_getForeignWitnessTable, C_CC,
               ProtocolDescriptorPtrTy),
          ATTRS(NoUnwind, ReadNone))
 
+// MetadataResponse swift_getInPlaceMetadata(MetadataRequest request,
+//                                           TypeContextDescriptor *type);
+FUNCTION(GetInPlaceMetadata, swift_getInPlaceMetadata, SwiftCC,
+         RETURNS(TypeMetadataResponseTy),
+         ARGS(SizeTy, TypeContextDescriptorPtrTy),
+         ATTRS(NoUnwind, ReadNone))
+
 // MetadataResponse swift_getGenericMetadata(MetadataRequest request,
 //                                           const void * const *arguments,
 //                                           TypeContextDescriptor *type);

stdlib/public/runtime/Metadata.cpp

@@ -175,6 +175,14 @@ areAllTransitiveMetadataComplete_cheap(const Metadata *metadata);
 static MetadataDependency
 checkTransitiveCompleteness(const Metadata *metadata);
 
+static PrivateMetadataState inferStateForMetadata(Metadata *metadata) {
+  if (metadata->getValueWitnesses()->isIncomplete())
+    return PrivateMetadataState::Abstract;
+
+  // TODO: internal vs. external layout-complete?
+  return PrivateMetadataState::LayoutComplete;
+}
+
 namespace {
   struct GenericCacheEntry final :
       VariadicMetadataCacheEntryBase<GenericCacheEntry> {
@@ -212,14 +220,6 @@ namespace {
       return { metadata, state };
     }
 
-    PrivateMetadataState inferStateForMetadata(Metadata *metadata) {
-      if (metadata->getValueWitnesses()->isIncomplete())
-        return PrivateMetadataState::Abstract;
-
-      // TODO: internal vs. external layout-complete?
-      return PrivateMetadataState::LayoutComplete;
-    }
-
     static const TypeContextDescriptor *getDescription(Metadata *type) {
       if (auto classType = dyn_cast<ClassMetadata>(type))
         return classType->getDescription();
@@ -533,6 +533,169 @@ swift::swift_getGenericMetadata(MetadataRequest request,
   return result.second;
 }
 
+/***************************************************************************/
+/*** In-place metadata initialization **************************************/
+/***************************************************************************/
+
+namespace {
+  /// A cache entry for "in-place" metadata initializations.
+  class InPlaceMetadataCacheEntry final
+      : public MetadataCacheEntryBase<InPlaceMetadataCacheEntry, int> {
+    ValueType Value = nullptr;
+
+    friend MetadataCacheEntryBase;
+    ValueType getValue() {
+      return Value;
+    }
+    void setValue(ValueType value) {
+      Value = value;
+    }
+
+  public:
+    // We have to give MetadataCacheEntryBase a non-empty list of trailing
+    // objects or else it gets annoyed.
+    static size_t numTrailingObjects(OverloadToken<int>) { return 0; }
+
+    static const char *getName() { return "InPlaceMetadataCache"; }
+
+    InPlaceMetadataCacheEntry() {}
+
+    AllocationResult allocate(const TypeContextDescriptor *description) {
+      auto valueTypeDescriptor = cast<ValueTypeDescriptor>(description);
+      auto &initialization =
+        valueTypeDescriptor->getInPlaceMetadataInitialization();
+
+      auto metadata = initialization.IncompleteMetadata.get();
+
+      auto state = inferStateForMetadata(metadata);
+      return { metadata, state };
+    }
+
+    static const TypeContextDescriptor *getDescription(Metadata *type) {
+      return cast<ValueMetadata>(type)->getDescription();
+    }
+
+    TryInitializeResult tryInitialize(Metadata *metadata,
+                                      PrivateMetadataState state,
+                               PrivateMetadataCompletionContext *context) {
+      assert(state != PrivateMetadataState::Complete);
+
+      // Finish the completion function.
+      if (state < PrivateMetadataState::NonTransitiveComplete) {
+        // Find a pattern.  Currently we always use the default pattern.
+        auto &initialization =
+          cast<ValueMetadata>(metadata)->getDescription()
+                                       ->getInPlaceMetadataInitialization();
+
+        // Complete the metadata's instantiation.
+        auto dependency =
+          initialization.CompletionFunction(metadata, &context->Public,
+                                            /*pattern*/ nullptr);
+
+        // If this failed with a dependency, infer the current metadata state
+        // and return.
+        if (dependency) {
+          return { inferStateForMetadata(metadata), dependency };
+        }
+      }
+
+      // Check for transitive completeness.
+      if (auto dependency = checkTransitiveCompleteness(metadata)) {
+        return { PrivateMetadataState::NonTransitiveComplete, dependency };
+      }
+
+      // We're done.
+      publishCompleteMetadata(metadata);
+      return { PrivateMetadataState::Complete, MetadataDependency() };
+    }
+
+    void publishCompleteMetadata(Metadata *metadata) {
+      auto &init = cast<ValueMetadata>(metadata)->getDescription()
+                                           ->getInPlaceMetadataInitialization();
+      auto &cache = *init.InitializationCache.get();
+      cache.Metadata.store(metadata, std::memory_order_release);
+    }
+  };
+
+  /// An implementation of LockingConcurrentMapStorage that's more
+  /// appropriate for the in-place metadata cache.
+  ///
+  /// TODO: delete the cache entry when initialization is complete.
+  class InPlaceMetadataCacheStorage {
+    ConcurrencyControl Concurrency;
+
+  public:
+    using KeyType = const TypeContextDescriptor *;
+    using EntryType = InPlaceMetadataCacheEntry;
+
+    ConcurrencyControl &getConcurrency() { return Concurrency; }
+
+    template <class... ArgTys>
+    std::pair<EntryType*, bool>
+    getOrInsert(KeyType key, ArgTys &&...args) {
+      auto &init =
+        cast<ValueTypeDescriptor>(key)->getInPlaceMetadataInitialization();
+      auto &cache = *init.InitializationCache.get();
+
+      // Check for an existing entry.
+      auto existingEntry = cache.Private.load(std::memory_order_acquire);
+
+      // If there isn't one there, optimistically create an entry and
+      // try to swap it in.
+      if (!existingEntry) {
+        auto allocatedEntry = new InPlaceMetadataCacheEntry();
+        if (cache.Private.compare_exchange_strong(existingEntry,
+                                                  allocatedEntry,
+                                                  std::memory_order_acq_rel,
+                                                  std::memory_order_acquire)) {
+          // If that succeeded, return the entry we allocated and tell the
+          // caller we allocated it.
+          return { allocatedEntry, true };
+        }
+
+        // Otherwise, use the new entry and destroy the one we allocated.
+        assert(existingEntry && "spurious failure of strong compare-exchange?");
+        delete allocatedEntry;
+      }
+
+      return { static_cast<InPlaceMetadataCacheEntry*>(existingEntry), false };
+    }
+
+    EntryType *find(KeyType key) {
+      auto &init =
+        cast<ValueTypeDescriptor>(key)->getInPlaceMetadataInitialization();
+
+      return static_cast<InPlaceMetadataCacheEntry*>(
+        init.InitializationCache->Private.load(std::memory_order_acquire));
+    }
+
+    /// A default implementation for resolveEntry that assumes that the
+    /// key type is a lookup key for the map.
+    EntryType *resolveExistingEntry(KeyType key) {
+      auto entry = find(key);
+      assert(entry && "entry doesn't already exist!");
+      return entry;
+    }
+  };
+
+  class InPlaceMetadataCache
+      : public LockingConcurrentMap<InPlaceMetadataCacheEntry,
+                                    InPlaceMetadataCacheStorage> {
+  };
+} // end anonymous namespace
+
+/// The cache of all in-place metadata initializations.
+static Lazy<InPlaceMetadataCache> InPlaceMetadata;
+
+MetadataResponse
+swift::swift_getInPlaceMetadata(MetadataRequest request,
+                                const TypeContextDescriptor *description) {
+  auto result = InPlaceMetadata.get().getOrInsert(description, request,
+                                                  description);
+
+  return result.second;
+}
+
 /***************************************************************************/
 /*** Objective-C class wrappers ********************************************/
 /***************************************************************************/
@@ -3470,6 +3633,10 @@ static Result performOnMetadataCache(const Metadata *metadata,
   }
 
   if (!description->isGeneric()) {
+    if (description->hasInPlaceMetadataInitialization()) {
+      return std::move(callbacks).forInPlaceMetadata(description);
+    }
+
     return std::move(callbacks).forOtherMetadata(metadata);
   }
 
@@ -3498,6 +3665,10 @@ bool swift::addToMetadataQueue(MetadataCompletionQueueEntry *queueEntry,
       return cache.enqueue(key, QueueEntry, Dependency);
     }
 
+    bool forInPlaceMetadata(const TypeContextDescriptor *description) && {
+      return InPlaceMetadata.get().enqueue(description, QueueEntry, Dependency);
+    }
+
     bool forTupleMetadata(const TupleTypeMetadata *metadata) {
       return TupleTypes.get().enqueue(metadata, QueueEntry, Dependency);
     }
@@ -3521,6 +3692,10 @@ void swift::resumeMetadataCompletion(MetadataCompletionQueueEntry *queueEntry) {
       cache.resumeInitialization(key, QueueEntry);
     }
 
+    void forInPlaceMetadata(const TypeContextDescriptor *description) && {
+      InPlaceMetadata.get().resumeInitialization(description, QueueEntry);
+    }
+
     void forTupleMetadata(const TupleTypeMetadata *metadata) {
       TupleTypes.get().resumeInitialization(metadata, QueueEntry);
     }
@@ -3547,6 +3722,11 @@ MetadataResponse swift::swift_checkMetadataState(MetadataRequest request,
       return cache.await(key, Request);
     }
 
+    MetadataResponse forInPlaceMetadata(
+                                  const TypeContextDescriptor *description) && {
+      return InPlaceMetadata.get().await(description, Request);
+    }
+
     MetadataResponse forTupleMetadata(const TupleTypeMetadata *metadata) {
       return TupleTypes.get().await(metadata, Request);
     }
@@ -3630,6 +3810,11 @@ areAllTransitiveMetadataComplete_cheap(const Metadata *type) {
         return true;
       }
 
+      bool forInPlaceMetadata(const TypeContextDescriptor *description) && {
+        // TODO: this could be cheap enough.
+        return true;
+      }
+
       bool forTupleMetadata(const TupleTypeMetadata *metadata) {
         // TODO: this could be cheap enough.
         return true;
@@ -3787,6 +3972,11 @@ checkMetadataDependency(MetadataDependency dependency) {
       return cache.checkDependency(key, Requirement);
     }
 
+    MetadataDependency forInPlaceMetadata(
+                              const TypeContextDescriptor *description) && {
+      return InPlaceMetadata.get().checkDependency(description, Requirement);
+    }
+
     MetadataDependency forTupleMetadata(const TupleTypeMetadata *metadata) {
       return TupleTypes.get().checkDependency(metadata, Requirement);
     }