Complete support for outline existential storage

... or so I believe

include/swift/Runtime/RuntimeFunctions.def

@@ -236,6 +236,14 @@ FUNCTION(NativeStrongReleaseDirect, Swift, swift_releaseDirect, SwiftDirectRR_CC
          EFFECT(RuntimeEffect::RefCounting, RuntimeEffect::Deallocating),
          UNKNOWN_MEMEFFECTS)
 
+// void swift_releaseBox(void *ptr);
+FUNCTION(ReleaseBox, Swift, swift_releaseBox, C_CC, AlwaysAvailable,
+         RETURNS(VoidTy),
+         ARGS(RefCountedPtrTy),
+         ATTRS(NoUnwind),
+         EFFECT(RuntimeEffect::RefCounting, RuntimeEffect::Deallocating),
+         UNKNOWN_MEMEFFECTS)
+
 // void *swift_retain_n(void *ptr, int32_t n);
 FUNCTION(NativeStrongRetainN, Swift, swift_retain_n, C_CC, AlwaysAvailable,
          RETURNS(RefCountedPtrTy),

lib/IRGen/GenExistential.cpp

@@ -2314,7 +2314,21 @@ Address irgen::emitAllocateBoxedOpaqueExistentialBuffer(
     if (fixedTI->getFixedPacking(IGF.IGM) == FixedPacking::OffsetZero) {
       return valueTI.getAddressForPointer(IGF.Builder.CreateBitCast(
           existentialBuffer.getAddress(), IGF.IGM.PtrTy));
+    } else if (IGF.IGM.Context.LangOpts.hasFeature(Feature::EmbeddedExistentials)) {
+      llvm::Value *box, *address;
+      auto *metadata = existLayout.loadMetadataRef(IGF, existentialContainer);
+      IGF.emitAllocBoxCall(metadata, box, address);
+      llvm::Value *addressInBox =
+        IGF.Builder.CreateBitCast(address, IGF.IGM.OpaquePtrTy);
+      IGF.Builder.CreateStore(
+              box, Address(IGF.Builder.CreateBitCast(
+                               existentialBuffer.getAddress(), IGF.IGM.PtrTy),
+                           IGF.IGM.RefCountedPtrTy,
+                           existLayout.getAlignment(IGF.IGM)));
+
+      return valueTI.getAddressForPointer(addressInBox);
     }
+
     // Otherwise, allocate a box with enough storage.
     Address addr = emitAllocateExistentialBoxInBuffer(
         IGF, valueType, existentialBuffer, genericEnv, "exist.box.addr",
@@ -2883,7 +2897,11 @@ static llvm::Function *getDestroyBoxedOpaqueExistentialBufferFunction(
               Builder.CreateBitCast(buffer.getAddress(), IGM.PtrTy);
           auto *reference = Builder.CreateLoad(Address(
               referenceAddr, IGM.RefCountedPtrTy, buffer.getAlignment()));
-          IGF.emitNativeStrongRelease(reference, IGF.getDefaultAtomicity());
+          if (IGF.IGM.Context.LangOpts
+                .hasFeature(Feature::EmbeddedExistentials)) {
+            IGF.emitReleaseBox(reference);
+          } else
+            IGF.emitNativeStrongRelease(reference, IGF.getDefaultAtomicity());
 
           Builder.CreateRetVoid();
         }

lib/IRGen/GenHeap.cpp

@@ -1274,6 +1274,12 @@ void IRGenFunction::emitNativeStrongRelease(llvm::Value *value,
   emitUnaryRefCountCall(*this, function, value);
 }
 
+void IRGenFunction::emitReleaseBox(llvm::Value *value) {
+  if (doesNotRequireRefCounting(value))
+    return;
+  emitUnaryRefCountCall(*this, IGM.getReleaseBoxFn(), value);
+}
+
 void IRGenFunction::emitNativeSetDeallocating(llvm::Value *value) {
   if (doesNotRequireRefCounting(value)) return;
   emitUnaryRefCountCall(*this, IGM.getNativeSetDeallocatingFn(), value);

lib/IRGen/IRGenFunction.h

@@ -626,6 +626,9 @@ public:
   void emitNativeStrongRelease(llvm::Value *value, Atomicity atomicity);
   void emitNativeSetDeallocating(llvm::Value *value);
 
+  // Routines to deal with box (embedded) runtime calls.
+  void emitReleaseBox(llvm::Value *value);
+
   // Routines for the ObjC reference-counting style.
   void emitObjCStrongRetain(llvm::Value *value);
   llvm::Value *emitObjCRetainCall(llvm::Value *value);

stdlib/public/SwiftShims/swift/shims/EmbeddedShims.h

@@ -113,20 +113,42 @@ typedef struct {
 #endif
 } EmbeddedMetaDataPrefix;
 
-static inline __swift_size_t _swift_embedded_metadata_get_size(void *metadata) {
+static inline __swift_size_t
+_swift_embedded_metadata_get_size(void *metadata) {
   EmbeddedMetaDataPrefix *fullmeta = (EmbeddedMetaDataPrefix*)&((void **)metadata)[-1];
   return fullmeta->vwt->size;
 }
 
-static inline __swift_size_t _swift_embedded_metadata_get_align_mask(void *metadata) {
-  EmbeddedMetaDataPrefix *fullmeta = (EmbeddedMetaDataPrefix*)&((void **)metadata)[-1];
-
-  unsigned flags =  fullmeta->vwt->flags;
+static inline __swift_size_t
+_swift_embedded_metadata_get_align_mask_impl(EmbeddedMetaDataPrefix *fullMetadata) {
+  unsigned flags =  fullMetadata->vwt->flags;
   unsigned embeddedValueWitnessTableFlagsMask = 0xFF;
 
   return flags & embeddedValueWitnessTableFlagsMask;
 }
 
+static inline __swift_size_t
+_swift_embedded_metadata_get_align_mask(void *metadata) {
+  EmbeddedMetaDataPrefix *fullmeta = (EmbeddedMetaDataPrefix*)&((void **)metadata)[-1];
+  return _swift_embedded_metadata_get_align_mask_impl(fullmeta);
+}
+
+static inline void
+_swift_embedded_invoke_box_destroy(void *object) {
+  void *metadata = ((EmbeddedHeapObject *)object)->metadata;
+  EmbeddedMetaDataPrefix *fullmeta = (EmbeddedMetaDataPrefix*)&((void **)metadata)[-1];
+  __swift_size_t alignMask = _swift_embedded_metadata_get_align_mask_impl(fullmeta);
+  __swift_size_t headerSize = sizeof(void*) + sizeof(__swift_size_t);
+  __swift_size_t startOfBoxedValue = (headerSize + alignMask) & ~alignMask;
+  void *addrInBox = (void *)(((unsigned char *)object) + startOfBoxedValue);
+  fullmeta->vwt->destroyFn(addrInBox, metadata);
+}
+
+static inline void
+_swift_embedded_initialize_box(void *metadata, void *newObjectAddr, void *oldObjectAddr) {
+  EmbeddedMetaDataPrefix *fullmeta = (EmbeddedMetaDataPrefix*)&((void **)metadata)[-1];
+  fullmeta->vwt->initializeWithCopyFn(newObjectAddr, oldObjectAddr, metadata);
+}
 
 #ifdef __cplusplus
 } // extern "C"

stdlib/public/core/EmbeddedRuntime.swift

@@ -263,7 +263,7 @@ public func swift_allocEmptyBox() -> Builtin.RawPointer {
 
 
 @_silgen_name("swift_allocBox")
-public func swift_allocBox(metadata: Builtin.RawPointer) -> (Builtin.RawPointer, Builtin.RawPointer) {
+public func swift_allocBox(_ metadata: Builtin.RawPointer) -> (Builtin.RawPointer, Builtin.RawPointer) {
   let alignMask = Int(unsafe _swift_embedded_metadata_get_align_mask(UnsafeMutableRawPointer(metadata)))
   let size = Int(unsafe _swift_embedded_metadata_get_size(UnsafeMutableRawPointer(metadata)))
   let headerSize = unsafe MemoryLayout<Int>.size + MemoryLayout<UnsafeRawPointer>.size
@@ -283,12 +283,29 @@ public func swift_allocBox(metadata: Builtin.RawPointer) -> (Builtin.RawPointer,
   return (object._rawValue, boxedValueAddr._rawValue)
 }
 
-@_cdecl("swift_deallocBox")
-public func swift_deallocBox(object: Builtin.RawPointer) {
+@c
+public func swift_deallocBox(_ object: Builtin.RawPointer) {
   unsafe free(UnsafeMutableRawPointer(object))
 }
 
+@_silgen_name("swift_makeBoxUnique")
+public func swifft_makeBoxUnique(buffer: Builtin.RawPointer, metadata: Builtin.RawPointer, alignMask: Int) -> (Builtin.RawPointer, Builtin.RawPointer){
+  let addrOfHeapObjectPtr = unsafe UnsafeMutablePointer<Builtin.RawPointer>(buffer)
+  let box = unsafe addrOfHeapObjectPtr.pointee
+  let headerSize = unsafe MemoryLayout<Int>.size + MemoryLayout<UnsafeRawPointer>.size
+  let startOfBoxedValue = ((headerSize + alignMask) & ~alignMask)
+  let oldObjectAddr = unsafe UnsafeMutableRawPointer(box) + startOfBoxedValue
 
+  if !swift_isUniquelyReferenced_native(object: box) {
+    let refAndObjectAddr = swift_allocBox(metadata)
+    unsafe _swift_embedded_initialize_box(UnsafeMutableRawPointer(metadata), UnsafeMutableRawPointer(refAndObjectAddr.1), oldObjectAddr)
+    swift_releaseBox(box)
+    unsafe addrOfHeapObjectPtr.pointee = refAndObjectAddr.0
+    return refAndObjectAddr
+  } else {
+    return (box, oldObjectAddr._rawValue)
+  }
+}
 
 /// Refcounting
 
@@ -416,7 +433,7 @@ public func swift_release_n(object: Builtin.RawPointer, n: UInt32) {
   unsafe swift_release_n_(object: o, n: n)
 }
 
-func swift_release_n_(object: UnsafeMutablePointer<HeapObject>?, n: UInt32) {
+func swift_release_n_(object: UnsafeMutablePointer<HeapObject>?, n: UInt32, isBoxRelease: Bool = false) {
   guard let object = unsafe object else {
     return
   }
@@ -441,12 +458,25 @@ func swift_release_n_(object: UnsafeMutablePointer<HeapObject>?, n: UInt32) {
     let doNotFree = unsafe (loadedRefcount & HeapObject.doNotFreeBit) != 0
     unsafe storeRelaxed(refcount, newValue: HeapObject.immortalRefCount | (doNotFree ? HeapObject.doNotFreeBit : 0))
 
-    unsafe _swift_embedded_invoke_heap_object_destroy(object)
+    if isBoxRelease {
+        unsafe _swift_embedded_invoke_box_destroy(object)
+    } else {
+        unsafe _swift_embedded_invoke_heap_object_destroy(object)
+    }
   } else if resultingRefcount < 0 {
     fatalError("negative refcount")
   }
 }
 
+@c
+public func swift_releaseBox(_ object: Builtin.RawPointer) {
+  if !isValidPointerForNativeRetain(object: object) {
+    fatalError("not a valid pointer for releaseBox")
+  }
+  let o = unsafe UnsafeMutablePointer<HeapObject>(object)
+  unsafe swift_release_n_(object: o, n: 1, isBoxRelease: true)
+}
+
 @c
 public func swift_bridgeObjectRelease(object: Builtin.RawPointer) {
   swift_bridgeObjectRelease_n(object: object, n: 1)

test/embedded/existential.swift

@@ -57,6 +57,11 @@ enum GenericEnumWithClass<T> {
 // OUTPUT:  deinit called
 // OUTPUT:  deinit called
 
+// OUTPUT:  deinit called
+// OUTPUT:  deinit called
+// OUTPUT:  deinit called
+// OUTPUT:  deinit called
+
 // OUTPUT-NOT:  deinit called
 
 func test() {
@@ -68,6 +73,8 @@ func test() {
     let _: any Any = GenericEnumWithClass.c(GC<Int>())
     let _: any Any = (3, 4)
     let _: any Any = (StructWithClass(), StructWithClass())
+    // outline storage case
+    let _: any Any = (StructWithClass(), StructWithClass(), StructWithClass(), StructWithClass())
 }
 
 protocol Basic  {
@@ -94,6 +101,14 @@ struct MyStruct : Derived {
   func b() { print("b MyStruct") }
 }
 
+struct LargeMyStruct : Derived {
+  var x = (1, 2, 3, 4, 5)
+  var refCounted = StructWithClass()
+
+  func a() { print("a LargeMyStruct \(self.x.4)") }
+  func b() { print("b LargeMyStruct") }
+}
+
 enum MyEnum : Derived {
     case a
     case b(Int)
@@ -120,6 +135,7 @@ func test2(_ p: any Derived) {
 
 protocol ValuePrinter {
     func printValue()
+    mutating func mutate()
 }
 protocol WithAssoc {
     associatedtype Assoc : ValuePrinter
@@ -130,6 +146,20 @@ extension Int : ValuePrinter {
     func printValue() {
         print("my value: \(self)")
     }
+    mutating func mutate() {
+        self = 8
+        print("my value (mutating expect 8): \(self)")
+    }
+}
+
+extension LargeMyStruct : ValuePrinter {
+    func printValue() {
+        print("my value of LargeMyStruct: \(self.x.4)")
+    }
+    mutating func mutate() {
+        self.x = (6, 7, 8, 9, 10)
+        print("my value of LargeMyStruct (mutating expect 10): \(self.x.4)")
+    }
 }
 
 struct ConformWithAssoc : WithAssoc {
@@ -139,30 +169,56 @@ struct ConformWithAssoc : WithAssoc {
   }
 }
 
+struct ConformWithLargeAssoc : WithAssoc {
+  var x = LargeMyStruct()
+
+  func a() -> LargeMyStruct {
+    return x
+  }
+}
+
 func test3(_ p: any WithAssoc) {
   let x = p.a()
   x.printValue()
 }
 
+func test4(_ p: any WithAssoc) {
+  var x = p.a()
+  let c = x
+  x.mutate()
+  c.printValue()
+}
+
 @main
 struct Main {
   static func main() {
     test()
 
+    test2(Implementor())
 // OUTPUT: a
 // OUTPUT: b
+    test2(5)
 // OUTPUT: a Int 5
 // OUTPUT: b Int 5
+    test2(MyStruct())
 // OUTPUT: a MyStruct 5
 // OUTPUT: b MyStruct
+    test2(MyEnum.b(5))
 // OUTPUT: a MyEnum
 // OUTPUT: 5
 // OUTPUT: b MyEnum
-// OUTPUT: my value: 1
-    test2(Implementor())
-    test2(5)
-    test2(MyStruct())
-    test2(MyEnum.b(5))
     test3(ConformWithAssoc())
+// OUTPUT: my value: 1
+    test3(ConformWithLargeAssoc())
+// OUTPUT: my value of LargeMyStruct: 5
+// OUTPUT:  deinit called
+    test4(ConformWithAssoc())
+// OUTPUT: my value (mutating expect 8): 8
+// OUTPUT: my value: 1
+    test4(ConformWithLargeAssoc())
+// OUTPUT: my value of LargeMyStruct (mutating expect 10): 10
+// OUTPUT: my value of LargeMyStruct: 5
+// OUTPUT:  deinit called
+// OUTPUT-NOT:  deinit called
   }
 }