Support in-place value metadata initialization in the runtime.

stdlib/public/runtime/Metadata.cpp

@@ -175,6 +175,14 @@ areAllTransitiveMetadataComplete_cheap(const Metadata *metadata);
 static MetadataDependency
 checkTransitiveCompleteness(const Metadata *metadata);
 
+static PrivateMetadataState inferStateForMetadata(Metadata *metadata) {
+  if (metadata->getValueWitnesses()->isIncomplete())
+    return PrivateMetadataState::Abstract;
+
+  // TODO: internal vs. external layout-complete?
+  return PrivateMetadataState::LayoutComplete;
+}
+
 namespace {
   struct GenericCacheEntry final :
       VariadicMetadataCacheEntryBase<GenericCacheEntry> {
@@ -212,14 +220,6 @@ namespace {
       return { metadata, state };
     }
 
-    PrivateMetadataState inferStateForMetadata(Metadata *metadata) {
-      if (metadata->getValueWitnesses()->isIncomplete())
-        return PrivateMetadataState::Abstract;
-
-      // TODO: internal vs. external layout-complete?
-      return PrivateMetadataState::LayoutComplete;
-    }
-
     static const TypeContextDescriptor *getDescription(Metadata *type) {
       if (auto classType = dyn_cast<ClassMetadata>(type))
         return classType->getDescription();
@@ -533,6 +533,169 @@ swift::swift_getGenericMetadata(MetadataRequest request,
   return result.second;
 }
 
+/***************************************************************************/
+/*** In-place metadata initialization **************************************/
+/***************************************************************************/
+
+namespace {
+  /// A cache entry for "in-place" metadata initializations.
+  class InPlaceMetadataCacheEntry final
+      : public MetadataCacheEntryBase<InPlaceMetadataCacheEntry, int> {
+    ValueType Value = nullptr;
+
+    friend MetadataCacheEntryBase;
+    ValueType getValue() {
+      return Value;
+    }
+    void setValue(ValueType value) {
+      Value = value;
+    }
+
+  public:
+    // We have to give MetadataCacheEntryBase a non-empty list of trailing
+    // objects or else it gets annoyed.
+    static size_t numTrailingObjects(OverloadToken<int>) { return 0; }
+
+    static const char *getName() { return "InPlaceMetadataCache"; }
+
+    InPlaceMetadataCacheEntry() {}
+
+    AllocationResult allocate(const TypeContextDescriptor *description) {
+      auto valueTypeDescriptor = cast<ValueTypeDescriptor>(description);
+      auto &initialization =
+        valueTypeDescriptor->getInPlaceMetadataInitialization();
+
+      auto metadata = initialization.IncompleteMetadata.get();
+
+      auto state = inferStateForMetadata(metadata);
+      return { metadata, state };
+    }
+
+    static const TypeContextDescriptor *getDescription(Metadata *type) {
+      return cast<ValueMetadata>(type)->getDescription();
+    }
+
+    TryInitializeResult tryInitialize(Metadata *metadata,
+                                      PrivateMetadataState state,
+                               PrivateMetadataCompletionContext *context) {
+      assert(state != PrivateMetadataState::Complete);
+
+      // Finish the completion function.
+      if (state < PrivateMetadataState::NonTransitiveComplete) {
+        // Find a pattern.  Currently we always use the default pattern.
+        auto &initialization =
+          cast<ValueMetadata>(metadata)->getDescription()
+                                       ->getInPlaceMetadataInitialization();
+
+        // Complete the metadata's instantiation.
+        auto dependency =
+          initialization.CompletionFunction(metadata, &context->Public,
+                                            /*pattern*/ nullptr);
+
+        // If this failed with a dependency, infer the current metadata state
+        // and return.
+        if (dependency) {
+          return { inferStateForMetadata(metadata), dependency };
+        }
+      }
+
+      // Check for transitive completeness.
+      if (auto dependency = checkTransitiveCompleteness(metadata)) {
+        return { PrivateMetadataState::NonTransitiveComplete, dependency };
+      }
+
+      // We're done.
+      publishCompleteMetadata(metadata);
+      return { PrivateMetadataState::Complete, MetadataDependency() };
+    }
+
+    void publishCompleteMetadata(Metadata *metadata) {
+      auto &init = cast<ValueMetadata>(metadata)->getDescription()
+                                           ->getInPlaceMetadataInitialization();
+      auto &cache = *init.InitializationCache.get();
+      cache.Metadata.store(metadata, std::memory_order_release);
+    }
+  };
+
+  /// An implementation of LockingConcurrentMapStorage that's more
+  /// appropriate for the in-place metadata cache.
+  ///
+  /// TODO: delete the cache entry when initialization is complete.
+  class InPlaceMetadataCacheStorage {
+    ConcurrencyControl Concurrency;
+
+  public:
+    using KeyType = const TypeContextDescriptor *;
+    using EntryType = InPlaceMetadataCacheEntry;
+
+    ConcurrencyControl &getConcurrency() { return Concurrency; }
+
+    template <class... ArgTys>
+    std::pair<EntryType*, bool>
+    getOrInsert(KeyType key, ArgTys &&...args) {
+      auto &init =
+        cast<ValueTypeDescriptor>(key)->getInPlaceMetadataInitialization();
+      auto &cache = *init.InitializationCache.get();
+
+      // Check for an existing entry.
+      auto existingEntry = cache.Private.load(std::memory_order_acquire);
+
+      // If there isn't one there, optimistically create an entry and
+      // try to swap it in.
+      if (!existingEntry) {
+        auto allocatedEntry = new InPlaceMetadataCacheEntry();
+        if (cache.Private.compare_exchange_strong(existingEntry,
+                                                  allocatedEntry,
+                                                  std::memory_order_acq_rel,
+                                                  std::memory_order_acquire)) {
+          // If that succeeded, return the entry we allocated and tell the
+          // caller we allocated it.
+          return { allocatedEntry, true };
+        }
+
+        // Otherwise, use the new entry and destroy the one we allocated.
+        assert(existingEntry && "spurious failure of strong compare-exchange?");
+        delete allocatedEntry;
+      }
+
+      return { static_cast<InPlaceMetadataCacheEntry*>(existingEntry), false };
+    }
+
+    EntryType *find(KeyType key) {
+      auto &init =
+        cast<ValueTypeDescriptor>(key)->getInPlaceMetadataInitialization();
+
+      return static_cast<InPlaceMetadataCacheEntry*>(
+        init.InitializationCache->Private.load(std::memory_order_acquire));
+    }
+
+    /// A default implementation for resolveEntry that assumes that the
+    /// key type is a lookup key for the map.
+    EntryType *resolveExistingEntry(KeyType key) {
+      auto entry = find(key);
+      assert(entry && "entry doesn't already exist!");
+      return entry;
+    }
+  };
+
+  class InPlaceMetadataCache
+      : public LockingConcurrentMap<InPlaceMetadataCacheEntry,
+                                    InPlaceMetadataCacheStorage> {
+  };
+} // end anonymous namespace
+
+/// The cache of all in-place metadata initializations.
+static Lazy<InPlaceMetadataCache> InPlaceMetadata;
+
+MetadataResponse
+swift::swift_getInPlaceMetadata(MetadataRequest request,
+                                const TypeContextDescriptor *description) {
+  auto result = InPlaceMetadata.get().getOrInsert(description, request,
+                                                  description);
+
+  return result.second;
+}
+
 /***************************************************************************/
 /*** Objective-C class wrappers ********************************************/
 /***************************************************************************/
@@ -3470,6 +3633,10 @@ static Result performOnMetadataCache(const Metadata *metadata,
   }
 
   if (!description->isGeneric()) {
+    if (description->hasInPlaceMetadataInitialization()) {
+      return std::move(callbacks).forInPlaceMetadata(description);
+    }
+
     return std::move(callbacks).forOtherMetadata(metadata);
   }
 
@@ -3498,6 +3665,10 @@ bool swift::addToMetadataQueue(MetadataCompletionQueueEntry *queueEntry,
       return cache.enqueue(key, QueueEntry, Dependency);
     }
 
+    bool forInPlaceMetadata(const TypeContextDescriptor *description) && {
+      return InPlaceMetadata.get().enqueue(description, QueueEntry, Dependency);
+    }
+
     bool forTupleMetadata(const TupleTypeMetadata *metadata) {
       return TupleTypes.get().enqueue(metadata, QueueEntry, Dependency);
     }
@@ -3521,6 +3692,10 @@ void swift::resumeMetadataCompletion(MetadataCompletionQueueEntry *queueEntry) {
       cache.resumeInitialization(key, QueueEntry);
     }
 
+    void forInPlaceMetadata(const TypeContextDescriptor *description) && {
+      InPlaceMetadata.get().resumeInitialization(description, QueueEntry);
+    }
+
     void forTupleMetadata(const TupleTypeMetadata *metadata) {
       TupleTypes.get().resumeInitialization(metadata, QueueEntry);
     }
@@ -3547,6 +3722,11 @@ MetadataResponse swift::swift_checkMetadataState(MetadataRequest request,
       return cache.await(key, Request);
     }
 
+    MetadataResponse forInPlaceMetadata(
+                                  const TypeContextDescriptor *description) && {
+      return InPlaceMetadata.get().await(description, Request);
+    }
+
     MetadataResponse forTupleMetadata(const TupleTypeMetadata *metadata) {
       return TupleTypes.get().await(metadata, Request);
     }
@@ -3630,6 +3810,11 @@ areAllTransitiveMetadataComplete_cheap(const Metadata *type) {
         return true;
       }
 
+      bool forInPlaceMetadata(const TypeContextDescriptor *description) && {
+        // TODO: this could be cheap enough.
+        return true;
+      }
+
       bool forTupleMetadata(const TupleTypeMetadata *metadata) {
         // TODO: this could be cheap enough.
         return true;
@@ -3787,6 +3972,11 @@ checkMetadataDependency(MetadataDependency dependency) {
       return cache.checkDependency(key, Requirement);
     }
 
+    MetadataDependency forInPlaceMetadata(
+                              const TypeContextDescriptor *description) && {
+      return InPlaceMetadata.get().checkDependency(description, Requirement);
+    }
+
     MetadataDependency forTupleMetadata(const TupleTypeMetadata *metadata) {
       return TupleTypes.get().checkDependency(metadata, Requirement);
     }