Merge pull request #76734 from jckarter/atomic-layout-runtime

Runtime: Set 'is not bitwise borrowable' bit for raw layout types.

include/swift/ABI/Metadata.h

@@ -2275,6 +2275,24 @@ public:
     if (!hasGeneralizationSignature()) return 0;
     return getGenSigHeader()->getArgumentLayoutSizeInWords();
   }
+  
+  bool isCopyable() const {
+    if (!hasGeneralizationSignature()) {
+      return true;
+    }
+    auto *reqts = getGenSigRequirements();
+    for (unsigned i = 0, e = getNumGenSigRequirements(); i < e; ++i) {
+      auto &reqt = reqts[i];
+      if (reqt.getKind() != GenericRequirementKind::InvertedProtocols) {
+        continue;
+      }
+      if (reqt.getInvertedProtocols()
+              .contains(InvertibleProtocolKind::Copyable)) {
+        return false;
+      }
+    }
+    return true;
+  }
 };
 using ExtendedExistentialTypeShape
   = TargetExtendedExistentialTypeShape<InProcess>;

include/swift/ABI/MetadataValues.h

@@ -1558,6 +1558,12 @@ enum class RawLayoutFlags : uintptr_t {
 
   /// Whether or not this raw layout type was declared 'movesAsLike'.
   MovesAsLike = 0x2,
+  
+  /// Whether this raw layout type is bitwise borrowable.
+  ///
+  /// No raw layout types are yet, but should we change our mind about that in the future,
+  /// this flag is here.
+  BitwiseBorrowable = 0x4,
 };
 static inline RawLayoutFlags operator|(RawLayoutFlags lhs,
                                        RawLayoutFlags rhs) {
@@ -1573,6 +1579,9 @@ static inline bool isRawLayoutArray(RawLayoutFlags flags) {
 static inline bool shouldRawLayoutMoveAsLike(RawLayoutFlags flags) {
   return uintptr_t(flags) & uintptr_t(RawLayoutFlags::MovesAsLike);
 }
+static inline bool isRawLayoutBitwiseBorrowable(RawLayoutFlags flags) {
+  return uintptr_t(flags) & uintptr_t(RawLayoutFlags::BitwiseBorrowable);
+}
 
 namespace SpecialPointerAuthDiscriminators {
   // All of these values are the stable string hash of the corresponding

stdlib/public/runtime/Enum.cpp

@@ -382,7 +382,7 @@ swift::swift_initEnumMetadataMultiPayload(EnumMetadata *enumType,
                                      const TypeLayout * const *payloadLayouts) {
   // Accumulate the layout requirements of the payloads.
   size_t payloadSize = 0, alignMask = 0;
-  bool isPOD = true, isBT = true;
+  bool isPOD = true, isBT = true, isBB = true;
   for (unsigned i = 0; i < numPayloads; ++i) {
     const TypeLayout *payloadLayout = payloadLayouts[i];
     payloadSize
@@ -390,6 +390,7 @@ swift::swift_initEnumMetadataMultiPayload(EnumMetadata *enumType,
     alignMask |= payloadLayout->flags.getAlignmentMask();
     isPOD &= payloadLayout->flags.isPOD();
     isBT &= payloadLayout->flags.isBitwiseTakable();
+    isBB &= payloadLayout->flags.isBitwiseBorrowable();
   }
   
   // Store the max payload size in the metadata.
@@ -418,6 +419,7 @@ swift::swift_initEnumMetadataMultiPayload(EnumMetadata *enumType,
                      .withAlignmentMask(alignMask)
                      .withPOD(isPOD)
                      .withBitwiseTakable(isBT)
+                     .withBitwiseBorrowable(isBB)
                      .withEnumWitnesses(true)
                      .withInlineStorage(ValueWitnessTable::isValueInline(
                          isBT, totalSize, alignMask + 1)),

stdlib/public/runtime/Metadata.cpp

@@ -2142,6 +2142,7 @@ static void performBasicLayout(TypeLayout &layout,
   size_t alignMask = layout.flags.getAlignmentMask();
   bool isPOD = layout.flags.isPOD();
   bool isBitwiseTakable = layout.flags.isBitwiseTakable();
+  bool isBitwiseBorrowable = layout.flags.isBitwiseBorrowable();
   for (unsigned i = 0; i != numElements; ++i) {
     auto &elt = elements[i];
 
@@ -2157,6 +2158,7 @@ static void performBasicLayout(TypeLayout &layout,
     alignMask = std::max(alignMask, eltLayout->flags.getAlignmentMask());
     if (!eltLayout->flags.isPOD()) isPOD = false;
     if (!eltLayout->flags.isBitwiseTakable()) isBitwiseTakable = false;
+    if (!eltLayout->flags.isBitwiseBorrowable()) isBitwiseBorrowable = false;
   }
   bool isInline =
       ValueWitnessTable::isValueInline(isBitwiseTakable, size, alignMask + 1);
@@ -2166,6 +2168,7 @@ static void performBasicLayout(TypeLayout &layout,
                      .withAlignmentMask(alignMask)
                      .withPOD(isPOD)
                      .withBitwiseTakable(isBitwiseTakable)
+                     .withBitwiseBorrowable(isBitwiseBorrowable)
                      .withInlineStorage(isInline);
   layout.extraInhabitantCount = 0;
   layout.stride = std::max(size_t(1), roundUpToAlignMask(size, alignMask));
@@ -3050,7 +3053,9 @@ void swift::swift_initRawStructMetadata(StructMetadata *structType,
   vwtable->stride = stride;
   vwtable->flags = ValueWitnessFlags()
                     .withAlignmentMask(alignMask)
-                    .withCopyable(false);
+                    .withCopyable(false)
+                    .withBitwiseTakable(true)
+                    .withBitwiseBorrowable(false);
   vwtable->extraInhabitantCount = extraInhabitantCount;
 }
 
@@ -3090,6 +3095,9 @@ void swift::swift_initRawStructMetadata2(StructMetadata *structType,
     vwtable->flags = vwtable->flags
                 .withBitwiseTakable(likeTypeLayout->flags.isBitwiseTakable());
   }
+  
+  vwtable->flags = vwtable->flags
+    .withBitwiseBorrowable(isRawLayoutBitwiseBorrowable(rawLayoutFlags));
 
   // If the calculated size of this raw layout type is available to be put in
   // value buffers, then set the inline storage bit if our like type is also
@@ -4454,27 +4462,44 @@ public:
 
 class OpaqueExistentialValueWitnessTableCacheEntry {
 public:
+  struct Key {
+    unsigned numWitnessTables : 31;
+    unsigned copyable : 1;
+    
+    bool operator==(struct Key k) {
+      return k.numWitnessTables == numWitnessTables
+        && k.copyable == copyable;
+    }
+  };
+
   ValueWitnessTable Data;
 
-  OpaqueExistentialValueWitnessTableCacheEntry(unsigned numTables);
+  OpaqueExistentialValueWitnessTableCacheEntry(Key key);
 
   unsigned getNumWitnessTables() const {
     return (Data.size - sizeof(OpaqueExistentialContainer))
               / sizeof(const WitnessTable *);
   }
+  
+  bool isCopyable() const {
+    return Data.flags.isCopyable();
+  }
 
   intptr_t getKeyIntValueForDump() {
     return getNumWitnessTables();
   }
 
-  bool matchesKey(unsigned key) const { return key == getNumWitnessTables(); }
+  bool matchesKey(Key key) const {
+    return key == Key{getNumWitnessTables(), isCopyable()};
+  }
 
   friend llvm::hash_code
   hash_value(const OpaqueExistentialValueWitnessTableCacheEntry &value) {
-    return llvm::hash_value(value.getNumWitnessTables());
+    return llvm::hash_value(
+      std::make_pair(value.getNumWitnessTables(), value.isCopyable()));
   }
 
-  static size_t getExtraAllocationSize(unsigned numTables) {
+  static size_t getExtraAllocationSize(Key key) {
     return 0;
   }
   size_t getExtraAllocationSize() const {
@@ -4562,19 +4587,24 @@ OpaqueExistentialValueWitnessTables;
 /// Instantiate a value witness table for an opaque existential container with
 /// the given number of witness table pointers.
 static const ValueWitnessTable *
-getOpaqueExistentialValueWitnesses(unsigned numWitnessTables) {
+getOpaqueExistentialValueWitnesses(unsigned numWitnessTables,
+                                   bool copyable) {
   // We pre-allocate a couple of important cases.
-  if (numWitnessTables == 0)
+  if (copyable) {
-    return &OpaqueExistentialValueWitnesses_0;
+    if (numWitnessTables == 0)
-  if (numWitnessTables == 1)
+      return &OpaqueExistentialValueWitnesses_0;
-    return &OpaqueExistentialValueWitnesses_1;
+    if (numWitnessTables == 1)
+      return &OpaqueExistentialValueWitnesses_1;
+  }
 
-  return &OpaqueExistentialValueWitnessTables.getOrInsert(numWitnessTables)
+  return &OpaqueExistentialValueWitnessTables
-                                             .first->Data;
+    .getOrInsert(OpaqueExistentialValueWitnessTableCacheEntry::Key{
+                  numWitnessTables, copyable})
+    .first->Data;
 }
 
 OpaqueExistentialValueWitnessTableCacheEntry::
-OpaqueExistentialValueWitnessTableCacheEntry(unsigned numWitnessTables) {
+OpaqueExistentialValueWitnessTableCacheEntry(Key key) {
   using Box = NonFixedOpaqueExistentialBox;
   using Witnesses = NonFixedValueWitnesses<Box, /*known allocated*/ true>;
 
@@ -4584,16 +4614,24 @@ OpaqueExistentialValueWitnessTableCacheEntry(unsigned numWitnessTables) {
 #define DATA_VALUE_WITNESS(LOWER_ID, UPPER_ID, TYPE)
 #include "swift/ABI/ValueWitness.def"
 
-  Data.size = Box::Container::getSize(numWitnessTables);
+  Data.size = Box::Container::getSize(key.numWitnessTables);
   Data.flags = ValueWitnessFlags()
-    .withAlignment(Box::Container::getAlignment(numWitnessTables))
+    .withAlignment(Box::Container::getAlignment(key.numWitnessTables))
     .withPOD(false)
     .withBitwiseTakable(true)
-    .withInlineStorage(false);
+    .withInlineStorage(false)
+    .withCopyable(key.copyable)
+    // Non-bitwise-takable values are always stored out-of-line in existentials,
+    // so the existential representation itself is always bitwise-takable.
+    // Noncopyable values however can be bitwise-takable without being
+    // bitwise-borrowable, so noncopyable existentials are not bitwise-borrowable
+    // in the general case.
+    .withBitwiseBorrowable(key.copyable);
   Data.extraInhabitantCount = Witnesses::numExtraInhabitants;
-  Data.stride = Box::Container::getStride(numWitnessTables);
+  Data.stride = Box::Container::getStride(key.numWitnessTables);
 
-  assert(getNumWitnessTables() == numWitnessTables);
+  assert(getNumWitnessTables() == key.numWitnessTables);
+  assert(isCopyable() == key.copyable);
 }
 
 static const ValueWitnessTable ClassExistentialValueWitnesses_1 =
@@ -4662,7 +4700,8 @@ static const ValueWitnessTable *
 getExistentialValueWitnesses(ProtocolClassConstraint classConstraint,
                              const Metadata *superclassConstraint,
                              unsigned numWitnessTables,
-                             SpecialProtocol special) {
+                             SpecialProtocol special,
+                             bool copyable) {
   // Use special representation for special protocols.
   switch (special) {
   case SpecialProtocol::Error:
@@ -4685,7 +4724,7 @@ getExistentialValueWitnesses(ProtocolClassConstraint classConstraint,
                                              numWitnessTables);
   case ProtocolClassConstraint::Any:
     assert(superclassConstraint == nullptr);
-    return getOpaqueExistentialValueWitnesses(numWitnessTables);
+    return getOpaqueExistentialValueWitnesses(numWitnessTables, copyable);
   }
 
   swift_unreachable("Unhandled ProtocolClassConstraint in switch.");
@@ -4928,7 +4967,8 @@ ExistentialCacheEntry::ExistentialCacheEntry(Key key) {
   Data.ValueWitnesses = getExistentialValueWitnesses(key.ClassConstraint,
                                                      key.SuperclassConstraint,
                                                      numWitnessTables,
-                                                     special);
+                                                     special,
+                                                     /*copyable*/ true);
   Data.Flags = ExistentialTypeFlags()
     .withNumWitnessTables(numWitnessTables)
     .withClassConstraint(key.ClassConstraint)
@@ -5164,6 +5204,7 @@ public:
 const ValueWitnessTable *
 ExtendedExistentialTypeCacheEntry::getOrCreateVWT(Key key) {
   auto shape = key.Shape;
+  bool copyable = shape->isCopyable();
 
   if (auto witnesses = shape->getSuggestedValueWitnesses())
     return witnesses;
@@ -5202,7 +5243,8 @@ ExtendedExistentialTypeCacheEntry::getOrCreateVWT(Key key) {
     return getExistentialValueWitnesses(ProtocolClassConstraint::Any,
                                         /*superclass*/ nullptr,
                                         wtableStorageSizeInWords,
-                                        SpecialProtocol::None);
+                                        SpecialProtocol::None,
+                                        copyable);
 
   case SpecialKind::ExplicitLayout:
     swift_unreachable("shape with explicit layout but no suggested VWT");
@@ -5214,7 +5256,8 @@ ExtendedExistentialTypeCacheEntry::getOrCreateVWT(Key key) {
     return getExistentialValueWitnesses(ProtocolClassConstraint::Class,
                                         /*superclass*/ nullptr,
                                         wtableStorageSizeInWords,
-                                        SpecialProtocol::None);
+                                        SpecialProtocol::None,
+                                        /*copyable*/ true);
 
   case SpecialKind::Metatype:
     // Existential metatypes don't store type metadata.

test/Runtime/bitwise-borrowable-generics.swift

@@ -0,0 +1,42 @@
+// RUN: %target-run-simple-swift(-enable-experimental-feature RawLayout)
+// REQUIRES: executable_test
+// UNSUPPORTED: use_os_stdlib
+// UNSUPPORTED: back_deployment_runtime
+
+import Synchronization
+
+struct NC<T>: ~Copyable { var x: T }
+
+@_rawLayout(like: T, movesAsLike)
+struct Test<T>: ~Copyable {}
+
+func isBitwiseBorrowable(_ type: any (~Copyable.Type)) -> Bool {
+    let metadataPtr = unsafeBitCast(type,
+                                    to: UnsafePointer<UnsafePointer<UInt>>.self)
+    let flags = metadataPtr[-1][10]
+    return flags & 0x0110_0000 == 0
+}
+
+func test(_ type: any (~Copyable.Type)) {
+    print("\(isBitwiseBorrowable(type))")
+}
+
+protocol P: ~Copyable {}
+
+// CHECK: begin
+print("begin")
+
+// CHECK-NEXT: true
+test(Int.self)
+// CHECK-NEXT: true
+test(Any.self)
+// CHECK-NEXT: true
+test(P.self)
+// CHECK-NEXT: true
+test(NC<Int>.self)
+// CHECK-NEXT: true
+test(NC<Any>.self)
+// CHECK-NEXT: false
+test(Test<Int>.self)
+// CHECK-NEXT: false
+test(Test<Any>.self)