From dadb51e708315cec33b80cb46ad846d44ab6ad67 Mon Sep 17 00:00:00 2001 From: John McCall Date: Tue, 24 Jul 2018 18:27:42 -0400 Subject: [PATCH] Support in-place value metadata initialization in the runtime. --- include/swift/ABI/Metadata.h | 108 ++++++++++- include/swift/ABI/MetadataValues.h | 12 ++ include/swift/Remote/MetadataReader.h | 15 +- include/swift/Runtime/Metadata.h | 7 + include/swift/Runtime/RuntimeFunctions.def | 7 + stdlib/public/runtime/Metadata.cpp | 206 ++++++++++++++++++++- 6 files changed, 340 insertions(+), 15 deletions(-) diff --git a/include/swift/ABI/Metadata.h b/include/swift/ABI/Metadata.h index a8d55ac80f4..586a8a80aa6 100644 --- a/include/swift/ABI/Metadata.h +++ b/include/swift/ABI/Metadata.h @@ -3364,6 +3364,14 @@ public: return TypeContextDescriptorFlags(this->Flags.getKindSpecificFlags()); } + /// Does this type have non-trivial "in place" metadata initialization? + /// + /// The type of the initialization-control structure differs by subclass, + /// so it doesn't appear here. + bool hasInPlaceMetadataInitialization() const { + return getTypeContextDescriptorFlags().hasInPlaceMetadataInitialization(); + } + const TargetTypeGenericContextDescriptorHeader & getFullGenericContextHeader() const; @@ -3709,10 +3717,49 @@ public: using ClassDescriptor = TargetClassDescriptor; +/// The cache structure for non-trivial initialization of singleton value +/// metadata. +template +struct TargetInPlaceValueMetadataCache { + /// The metadata pointer. Clients can do dependency-ordered loads + /// from this, and if they see a non-zero value, it's a Complete + /// metadata. + std::atomic> Metadata; + + /// The private cache data. + std::atomic> Private; +}; +using InPlaceValueMetadataCache = + TargetInPlaceValueMetadataCache; + +/// The control structure for performing non-trivial initialization of +/// singleton value metadata, which is required when e.g. a non-generic +/// value type has a resilient component type. +template +struct TargetInPlaceValueMetadataInitialization { + /// The initialization cache. Out-of-line because mutable. + TargetRelativeDirectPointer> + InitializationCache; + + /// The incomplete metadata. + TargetRelativeDirectPointer> + IncompleteMetadata; + + /// The completion function. The pattern will always be null. + TargetRelativeDirectPointer + CompletionFunction; +}; + template class TargetValueTypeDescriptor : public TargetTypeContextDescriptor { public: + using InPlaceMetadataInitialization = + TargetInPlaceValueMetadataInitialization; + + const InPlaceMetadataInitialization &getInPlaceMetadataInitialization() const; + static bool classof(const TargetContextDescriptor *cd) { return cd->getKind() == ContextDescriptorKind::Struct || cd->getKind() == ContextDescriptorKind::Enum; @@ -3724,16 +3771,30 @@ template class TargetStructDescriptor final : public TargetValueTypeDescriptor, public TrailingGenericContextObjects, - TargetTypeGenericContextDescriptorHeader> { + TargetTypeGenericContextDescriptorHeader, + TargetInPlaceValueMetadataInitialization> { +public: + using InPlaceMetadataInitialization = + TargetInPlaceValueMetadataInitialization; + private: using TrailingGenericContextObjects = TrailingGenericContextObjects, - TargetTypeGenericContextDescriptorHeader>; + TargetTypeGenericContextDescriptorHeader, + InPlaceMetadataInitialization>; using TrailingObjects = typename TrailingGenericContextObjects::TrailingObjects; friend TrailingObjects; + template + using OverloadToken = typename TrailingObjects::template OverloadToken; + + using TrailingGenericContextObjects::numTrailingObjects; + size_t numTrailingObjects(OverloadToken) const{ + return this->hasInPlaceMetadataInitialization() ? 1 : 0; + } + public: using TrailingGenericContextObjects::getGenericContext; using TrailingGenericContextObjects::getGenericContextHeader; @@ -3752,6 +3813,11 @@ public: /// its stored properties. bool hasFieldOffsetVector() const { return FieldOffsetVectorOffset != 0; } + const InPlaceMetadataInitialization &getInPlaceMetadataInitialization() const{ + assert(this->hasInPlaceMetadataInitialization()); + return *this->template getTrailingObjects(); + } + static constexpr int32_t getGenericArgumentOffset() { return TargetStructMetadata::getGenericArgumentOffset(); } @@ -3767,16 +3833,30 @@ template class TargetEnumDescriptor final : public TargetValueTypeDescriptor, public TrailingGenericContextObjects, - TargetTypeGenericContextDescriptorHeader> { + TargetTypeGenericContextDescriptorHeader, + TargetInPlaceValueMetadataInitialization> { +public: + using InPlaceMetadataInitialization = + TargetInPlaceValueMetadataInitialization; + private: using TrailingGenericContextObjects = TrailingGenericContextObjects, - TargetTypeGenericContextDescriptorHeader>; + TargetTypeGenericContextDescriptorHeader, + InPlaceMetadataInitialization>; using TrailingObjects = typename TrailingGenericContextObjects::TrailingObjects; friend TrailingObjects; + template + using OverloadToken = typename TrailingObjects::template OverloadToken; + + using TrailingGenericContextObjects::numTrailingObjects; + size_t numTrailingObjects(OverloadToken) const{ + return this->hasInPlaceMetadataInitialization() ? 1 : 0; + } + public: using TrailingGenericContextObjects::getGenericContext; using TrailingGenericContextObjects::getGenericContextHeader; @@ -3813,6 +3893,11 @@ public: return TargetEnumMetadata::getGenericArgumentOffset(); } + const InPlaceMetadataInitialization &getInPlaceMetadataInitialization() const{ + assert(this->hasInPlaceMetadataInitialization()); + return *this->template getTrailingObjects(); + } + static bool classof(const TargetContextDescriptor *cd) { return cd->getKind() == ContextDescriptorKind::Enum; } @@ -3901,6 +3986,21 @@ TargetTypeContextDescriptor::getGenericParams() const { } } +template +inline const TargetInPlaceValueMetadataInitialization & +TargetValueTypeDescriptor::getInPlaceMetadataInitialization() const { + switch (this->getKind()) { + case ContextDescriptorKind::Enum: + return llvm::cast>(this) + ->getInPlaceMetadataInitialization(); + case ContextDescriptorKind::Struct: + return llvm::cast>(this) + ->getInPlaceMetadataInitialization(); + default: + swift_runtime_unreachable("Not a value type descriptor."); + } +} + } // end namespace swift #pragma clang diagnostic pop diff --git a/include/swift/ABI/MetadataValues.h b/include/swift/ABI/MetadataValues.h index 4345f9e6328..a1e657030c0 100644 --- a/include/swift/ABI/MetadataValues.h +++ b/include/swift/ABI/MetadataValues.h @@ -1202,6 +1202,14 @@ class TypeContextDescriptorFlags : public FlagSet { /// declarations associated with the same declaration. IsSynthesizedRelatedEntity = 3, + /// Set if the type requires non-trivial but non-generic metadata + /// initialization. It may or may not be truly "in place" depending + /// on the kind of metadata. + /// + /// Currently only meaningful for value descriptors, but will be + /// extended to class descriptors. + HasInPlaceMetadataInitialization = 4, + /// Set if the context descriptor is includes metadata for dynamically /// constructing a class's vtables at metadata instantiation time. /// @@ -1237,6 +1245,10 @@ public: isSynthesizedRelatedEntity, setIsSynthesizedRelatedEntity) + FLAGSET_DEFINE_FLAG_ACCESSORS(HasInPlaceMetadataInitialization, + hasInPlaceMetadataInitialization, + setHasInPlaceMetadataInitialization) + FLAGSET_DEFINE_FLAG_ACCESSORS(Class_HasVTable, class_hasVTable, class_setHasVTable) diff --git a/include/swift/Remote/MetadataReader.h b/include/swift/Remote/MetadataReader.h index 73d43b3525f..ea8de7b9871 100644 --- a/include/swift/Remote/MetadataReader.h +++ b/include/swift/Remote/MetadataReader.h @@ -1050,8 +1050,10 @@ private: sizeof(flags))) return nullptr; + TypeContextDescriptorFlags typeFlags(flags.getKindSpecificFlags()); unsigned baseSize = 0; unsigned genericHeaderSize = sizeof(GenericContextDescriptorHeader); + unsigned inPlaceInitSize = 0; bool hasVTable = false; switch (auto kind = flags.getKind()) { case ContextDescriptorKind::Module: @@ -1067,16 +1069,23 @@ private: case ContextDescriptorKind::Class: baseSize = sizeof(TargetClassDescriptor); genericHeaderSize = sizeof(TypeGenericContextDescriptorHeader); - hasVTable = TypeContextDescriptorFlags(flags.getKindSpecificFlags()) - .class_hasVTable(); + hasVTable = typeFlags.class_hasVTable(); break; case ContextDescriptorKind::Enum: baseSize = sizeof(TargetEnumDescriptor); genericHeaderSize = sizeof(TypeGenericContextDescriptorHeader); + if (typeFlags.hasInPlaceMetadataInitialization()) { + inPlaceInitSize = + sizeof(TargetInPlaceValueMetadataInitialization); + } break; case ContextDescriptorKind::Struct: baseSize = sizeof(TargetStructDescriptor); genericHeaderSize = sizeof(TypeGenericContextDescriptorHeader); + if (typeFlags.hasInPlaceMetadataInitialization()) { + inPlaceInitSize = + sizeof(TargetInPlaceValueMetadataInitialization); + } break; case ContextDescriptorKind::Protocol: baseSize = sizeof(TargetProtocolDescriptorRef); @@ -1122,7 +1131,7 @@ private: + header.VTableSize * sizeof(TargetMethodDescriptor); } - unsigned size = baseSize + genericsSize + vtableSize; + unsigned size = baseSize + genericsSize + vtableSize + inPlaceInitSize; auto buffer = (uint8_t *)malloc(size); if (!Reader->readBytes(RemoteAddress(address), buffer, size)) { free(buffer); diff --git a/include/swift/Runtime/Metadata.h b/include/swift/Runtime/Metadata.h index e66c9ab0fef..2cf401fc4a3 100644 --- a/include/swift/Runtime/Metadata.h +++ b/include/swift/Runtime/Metadata.h @@ -335,6 +335,13 @@ ClassMetadataBounds getResilientMetadataBounds( const ClassDescriptor *descriptor); int32_t getResilientImmediateMembersOffset(const ClassDescriptor *descriptor); +/// \brief Fetch a uniqued metadata object for a nominal type which requires +/// in-place metadata initialization. +SWIFT_RUNTIME_EXPORT SWIFT_CC(swift) +MetadataResponse +swift_getInPlaceMetadata(MetadataRequest request, + const TypeContextDescriptor *description); + /// \brief Fetch a uniqued metadata object for a generic nominal type. SWIFT_RUNTIME_EXPORT SWIFT_CC(swift) MetadataResponse diff --git a/include/swift/Runtime/RuntimeFunctions.def b/include/swift/Runtime/RuntimeFunctions.def index 5e09b80478d..1a7ba9ec8fc 100644 --- a/include/swift/Runtime/RuntimeFunctions.def +++ b/include/swift/Runtime/RuntimeFunctions.def @@ -653,6 +653,13 @@ FUNCTION(GetForeignWitnessTable, swift_getForeignWitnessTable, C_CC, ProtocolDescriptorPtrTy), ATTRS(NoUnwind, ReadNone)) +// MetadataResponse swift_getInPlaceMetadata(MetadataRequest request, +// TypeContextDescriptor *type); +FUNCTION(GetInPlaceMetadata, swift_getInPlaceMetadata, SwiftCC, + RETURNS(TypeMetadataResponseTy), + ARGS(SizeTy, TypeContextDescriptorPtrTy), + ATTRS(NoUnwind, ReadNone)) + // MetadataResponse swift_getGenericMetadata(MetadataRequest request, // const void * const *arguments, // TypeContextDescriptor *type); diff --git a/stdlib/public/runtime/Metadata.cpp b/stdlib/public/runtime/Metadata.cpp index 8326f0b7976..4789db9791d 100644 --- a/stdlib/public/runtime/Metadata.cpp +++ b/stdlib/public/runtime/Metadata.cpp @@ -175,6 +175,14 @@ areAllTransitiveMetadataComplete_cheap(const Metadata *metadata); static MetadataDependency checkTransitiveCompleteness(const Metadata *metadata); +static PrivateMetadataState inferStateForMetadata(Metadata *metadata) { + if (metadata->getValueWitnesses()->isIncomplete()) + return PrivateMetadataState::Abstract; + + // TODO: internal vs. external layout-complete? + return PrivateMetadataState::LayoutComplete; +} + namespace { struct GenericCacheEntry final : VariadicMetadataCacheEntryBase { @@ -212,14 +220,6 @@ namespace { return { metadata, state }; } - PrivateMetadataState inferStateForMetadata(Metadata *metadata) { - if (metadata->getValueWitnesses()->isIncomplete()) - return PrivateMetadataState::Abstract; - - // TODO: internal vs. external layout-complete? - return PrivateMetadataState::LayoutComplete; - } - static const TypeContextDescriptor *getDescription(Metadata *type) { if (auto classType = dyn_cast(type)) return classType->getDescription(); @@ -533,6 +533,169 @@ swift::swift_getGenericMetadata(MetadataRequest request, return result.second; } +/***************************************************************************/ +/*** In-place metadata initialization **************************************/ +/***************************************************************************/ + +namespace { + /// A cache entry for "in-place" metadata initializations. + class InPlaceMetadataCacheEntry final + : public MetadataCacheEntryBase { + ValueType Value = nullptr; + + friend MetadataCacheEntryBase; + ValueType getValue() { + return Value; + } + void setValue(ValueType value) { + Value = value; + } + + public: + // We have to give MetadataCacheEntryBase a non-empty list of trailing + // objects or else it gets annoyed. + static size_t numTrailingObjects(OverloadToken) { return 0; } + + static const char *getName() { return "InPlaceMetadataCache"; } + + InPlaceMetadataCacheEntry() {} + + AllocationResult allocate(const TypeContextDescriptor *description) { + auto valueTypeDescriptor = cast(description); + auto &initialization = + valueTypeDescriptor->getInPlaceMetadataInitialization(); + + auto metadata = initialization.IncompleteMetadata.get(); + + auto state = inferStateForMetadata(metadata); + return { metadata, state }; + } + + static const TypeContextDescriptor *getDescription(Metadata *type) { + return cast(type)->getDescription(); + } + + TryInitializeResult tryInitialize(Metadata *metadata, + PrivateMetadataState state, + PrivateMetadataCompletionContext *context) { + assert(state != PrivateMetadataState::Complete); + + // Finish the completion function. + if (state < PrivateMetadataState::NonTransitiveComplete) { + // Find a pattern. Currently we always use the default pattern. + auto &initialization = + cast(metadata)->getDescription() + ->getInPlaceMetadataInitialization(); + + // Complete the metadata's instantiation. + auto dependency = + initialization.CompletionFunction(metadata, &context->Public, + /*pattern*/ nullptr); + + // If this failed with a dependency, infer the current metadata state + // and return. + if (dependency) { + return { inferStateForMetadata(metadata), dependency }; + } + } + + // Check for transitive completeness. + if (auto dependency = checkTransitiveCompleteness(metadata)) { + return { PrivateMetadataState::NonTransitiveComplete, dependency }; + } + + // We're done. + publishCompleteMetadata(metadata); + return { PrivateMetadataState::Complete, MetadataDependency() }; + } + + void publishCompleteMetadata(Metadata *metadata) { + auto &init = cast(metadata)->getDescription() + ->getInPlaceMetadataInitialization(); + auto &cache = *init.InitializationCache.get(); + cache.Metadata.store(metadata, std::memory_order_release); + } + }; + + /// An implementation of LockingConcurrentMapStorage that's more + /// appropriate for the in-place metadata cache. + /// + /// TODO: delete the cache entry when initialization is complete. + class InPlaceMetadataCacheStorage { + ConcurrencyControl Concurrency; + + public: + using KeyType = const TypeContextDescriptor *; + using EntryType = InPlaceMetadataCacheEntry; + + ConcurrencyControl &getConcurrency() { return Concurrency; } + + template + std::pair + getOrInsert(KeyType key, ArgTys &&...args) { + auto &init = + cast(key)->getInPlaceMetadataInitialization(); + auto &cache = *init.InitializationCache.get(); + + // Check for an existing entry. + auto existingEntry = cache.Private.load(std::memory_order_acquire); + + // If there isn't one there, optimistically create an entry and + // try to swap it in. + if (!existingEntry) { + auto allocatedEntry = new InPlaceMetadataCacheEntry(); + if (cache.Private.compare_exchange_strong(existingEntry, + allocatedEntry, + std::memory_order_acq_rel, + std::memory_order_acquire)) { + // If that succeeded, return the entry we allocated and tell the + // caller we allocated it. + return { allocatedEntry, true }; + } + + // Otherwise, use the new entry and destroy the one we allocated. + assert(existingEntry && "spurious failure of strong compare-exchange?"); + delete allocatedEntry; + } + + return { static_cast(existingEntry), false }; + } + + EntryType *find(KeyType key) { + auto &init = + cast(key)->getInPlaceMetadataInitialization(); + + return static_cast( + init.InitializationCache->Private.load(std::memory_order_acquire)); + } + + /// A default implementation for resolveEntry that assumes that the + /// key type is a lookup key for the map. + EntryType *resolveExistingEntry(KeyType key) { + auto entry = find(key); + assert(entry && "entry doesn't already exist!"); + return entry; + } + }; + + class InPlaceMetadataCache + : public LockingConcurrentMap { + }; +} // end anonymous namespace + +/// The cache of all in-place metadata initializations. +static Lazy InPlaceMetadata; + +MetadataResponse +swift::swift_getInPlaceMetadata(MetadataRequest request, + const TypeContextDescriptor *description) { + auto result = InPlaceMetadata.get().getOrInsert(description, request, + description); + + return result.second; +} + /***************************************************************************/ /*** Objective-C class wrappers ********************************************/ /***************************************************************************/ @@ -3470,6 +3633,10 @@ static Result performOnMetadataCache(const Metadata *metadata, } if (!description->isGeneric()) { + if (description->hasInPlaceMetadataInitialization()) { + return std::move(callbacks).forInPlaceMetadata(description); + } + return std::move(callbacks).forOtherMetadata(metadata); } @@ -3498,6 +3665,10 @@ bool swift::addToMetadataQueue(MetadataCompletionQueueEntry *queueEntry, return cache.enqueue(key, QueueEntry, Dependency); } + bool forInPlaceMetadata(const TypeContextDescriptor *description) && { + return InPlaceMetadata.get().enqueue(description, QueueEntry, Dependency); + } + bool forTupleMetadata(const TupleTypeMetadata *metadata) { return TupleTypes.get().enqueue(metadata, QueueEntry, Dependency); } @@ -3521,6 +3692,10 @@ void swift::resumeMetadataCompletion(MetadataCompletionQueueEntry *queueEntry) { cache.resumeInitialization(key, QueueEntry); } + void forInPlaceMetadata(const TypeContextDescriptor *description) && { + InPlaceMetadata.get().resumeInitialization(description, QueueEntry); + } + void forTupleMetadata(const TupleTypeMetadata *metadata) { TupleTypes.get().resumeInitialization(metadata, QueueEntry); } @@ -3547,6 +3722,11 @@ MetadataResponse swift::swift_checkMetadataState(MetadataRequest request, return cache.await(key, Request); } + MetadataResponse forInPlaceMetadata( + const TypeContextDescriptor *description) && { + return InPlaceMetadata.get().await(description, Request); + } + MetadataResponse forTupleMetadata(const TupleTypeMetadata *metadata) { return TupleTypes.get().await(metadata, Request); } @@ -3630,6 +3810,11 @@ areAllTransitiveMetadataComplete_cheap(const Metadata *type) { return true; } + bool forInPlaceMetadata(const TypeContextDescriptor *description) && { + // TODO: this could be cheap enough. + return true; + } + bool forTupleMetadata(const TupleTypeMetadata *metadata) { // TODO: this could be cheap enough. return true; @@ -3787,6 +3972,11 @@ checkMetadataDependency(MetadataDependency dependency) { return cache.checkDependency(key, Requirement); } + MetadataDependency forInPlaceMetadata( + const TypeContextDescriptor *description) && { + return InPlaceMetadata.get().checkDependency(description, Requirement); + } + MetadataDependency forTupleMetadata(const TupleTypeMetadata *metadata) { return TupleTypes.get().checkDependency(metadata, Requirement); }