[CoroutineAccessors] Add frame de/alloc functions.

To enable ABIs which store extra info in the frame, add two new slots to the coroutine allocator function table. For example, a frame could have a header containing a context pointer at a negative offset from the address returned from `swift_coro_alloc_frame`. The frame deallocation function would then know to deallocate more space correspondingly.
2025-12-14 20:36:38 +01:00 · 2025-10-17 17:03:49 -07:00
parent a2a4d22042
commit 76881b1d52
15 changed files with 237 additions and 66 deletions
--- a/include/swift/ABI/Coro.h
+++ b/include/swift/ABI/Coro.h
@@ -61,11 +61,15 @@ struct CoroAllocator;
 using CoroAllocation = void *;
 using CoroAllocateFn = CoroAllocation (*)(CoroAllocator *, size_t);
 using CoroDealllocateFn = void (*)(CoroAllocator *, CoroAllocation);
+using CoroAllocateFrameFn = CoroAllocation (*)(CoroAllocator *, size_t);
+using CoroDealllocateFrameFn = void (*)(CoroAllocator *, CoroAllocation);

 struct CoroAllocator {
  CoroAllocatorFlags flags;
  CoroAllocateFn allocate;
  CoroDealllocateFn deallocate;
+  CoroAllocateFrameFn allocateFrame;
+  CoroDealllocateFrameFn deallocateFrame;

  /// Whether the allocator should deallocate memory on calls to
  /// swift_coro_dealloc.
--- a/include/swift/ABI/MetadataValues.h
+++ b/include/swift/ABI/MetadataValues.h
@@ -1797,6 +1797,8 @@ namespace SpecialPointerAuthDiscriminators {
  /// Function pointers stored in the coro allocator struct.
  const uint16_t CoroAllocationFunction = 0x5f95;   // = 24469
  const uint16_t CoroDeallocationFunction = 0x9faf; // = 40879
+  const uint16_t CoroFrameAllocationFunction = 0xd251;   // = 53841
+  const uint16_t CoroFrameDeallocationFunction = 0x5ba8; // = 23464
 }

 /// The number of arguments that will be passed directly to a generic
--- a/include/swift/AST/IRGenOptions.h
+++ b/include/swift/AST/IRGenOptions.h
@@ -257,6 +257,14 @@ struct PointerAuthOptions : clang::PointerAuthOptions {

  /// Stored in a coro allocator struct, the function used to deallocate memory.
  PointerAuthSchema CoroDeallocationFunction;
+
+  /// Stored in a coro allocator struct, the function used to allocate a callee
+  /// coroutine's fixed-size frame.
+  PointerAuthSchema CoroFrameAllocationFunction;
+
+  /// Stored in a coro allocator struct, the function used to deallocate a
+  /// callee coroutine's fixed-size frame.
+  PointerAuthSchema CoroFrameDeallocationFunction;
 };

 enum class JITDebugArtifact : unsigned {
--- a/lib/IRGen/GenCall.cpp
+++ b/lib/IRGen/GenCall.cpp
@@ -5178,10 +5178,12 @@ void irgen::emitYieldOnce2CoroutineEntry(IRGenFunction &IGF,
  IGF.setCoroutineAllocator(allocator);
  auto allocFn = IGF.IGM.getOpaquePtr(getCoroAllocFn(IGF.IGM));
  auto deallocFn = IGF.IGM.getOpaquePtr(getCoroDeallocFn(IGF.IGM));
+  auto allocFrameFn = IGF.IGM.getOpaquePtr(getCoroAllocFrameFn(IGF.IGM));
+  auto deallocFrameFn = IGF.IGM.getOpaquePtr(getCoroDeallocFrameFn(IGF.IGM));
  emitRetconCoroutineEntry(
      IGF, fnType, buffer, llvm::Intrinsic::coro_id_retcon_once_dynamic,
      Size(-1) /*dynamic-to-IRGen size*/, IGF.IGM.getCoroStaticFrameAlignment(),
-      {cfp, allocator}, allocFn, deallocFn, {});
+      {cfp, allocator}, allocFn, deallocFn, {allocFrameFn, deallocFrameFn});
 }
 void irgen::emitYieldOnce2CoroutineEntry(
    IRGenFunction &IGF, LinkEntity coroFunction, CanSILFunctionType fnType,
@@ -5267,9 +5269,10 @@ StackAddress irgen::emitAllocCoroStaticFrame(IRGenFunction &IGF,
  // TODO: Avoid swift_task_alloc (async) and malloc (yield_once) if the
  //       suspension doesn't span an apply of an async function or a yield
  //       respectively.
-  auto retval =
-      IGF.emitDynamicAlloca(IGF.IGM.Int8Ty, size, Alignment(MaximumAlignment),
-                            AllowsTaskAlloc, "caller-coro-frame");
+  auto retval = IGF.emitDynamicAlloca(
+      IGF.IGM.Int8Ty, size, Alignment(MaximumAlignment), AllowsTaskAlloc,
+      IsForCalleeCoroutineFrame_t(IGF.isCalleeAllocatedCoroutine()),
+      "callee-coro-frame");
  IGF.Builder.CreateLifetimeStart(retval.getAddress(),
                                  Size(-1) /*dynamic size*/);
  return retval;
@@ -5277,7 +5280,8 @@ StackAddress irgen::emitAllocCoroStaticFrame(IRGenFunction &IGF,
 void irgen::emitDeallocCoroStaticFrame(IRGenFunction &IGF, StackAddress frame) {
  IGF.Builder.CreateLifetimeEnd(frame.getAddress(), Size(-1) /*dynamic size*/);
  IGF.emitDeallocateDynamicAlloca(frame, /*allowTaskAlloc*/ true,
-                                  /*useTaskDeallocThrough*/ true);
+                                  /*useTaskDeallocThrough*/ true,
+                                  /*forCalleeCoroutineFrame*/ true);
 }

 llvm::Value *irgen::emitYield(IRGenFunction &IGF,
--- a/lib/IRGen/GenCoro.cpp
+++ b/lib/IRGen/GenCoro.cpp
@@ -24,6 +24,7 @@
 #include "GenPointerAuth.h"
 #include "IRGenFunction.h"
 #include "IRGenModule.h"
+#include "clang/CodeGen/CodeGenABITypes.h"

 using namespace swift;
 using namespace irgen;
@@ -618,15 +619,46 @@ void IRGenFunction::emitTaskDeallocThrough(Address address) {
 }

 namespace {
+struct AllocationKind {
+  enum Kind {
+    Normal,
+    Frame,
+  };
+  Kind kind;
+  constexpr AllocationKind(Kind kind) : kind(kind) {}
+  constexpr operator Kind() { return kind; }
+
+  StringRef getAllocationFunctionName() {
+    switch (kind) {
+    case Normal:
+      return "_swift_coro_alloc";
+    case Frame:
+      return "_swift_coro_alloc_frame";
+    }
+  }
+
+  StringRef getDeallocationFunctionName() {
+    switch (kind) {
+    case Normal:
+      return "_swift_coro_dealloc";
+    case Frame:
+      return "_swift_coro_dealloc_frame";
+    }
+  }
+};
+
 struct Allocator {
  struct Field {
    enum Kind : uint8_t {
      Flags = 0,
      Allocate = 1,
      Deallocate = 2,
+      AllocateFrame = 3,
+      DeallocateFrame = 4,
    };
-    constexpr static std::array<Field, 3> allFields() {
-      return {Field::Flags, Field::Allocate, Field::Deallocate};
+    constexpr static std::array<Field, 5> allFields() {
+      return {Field::Flags, Field::Allocate, Field::Deallocate,
+              Field::AllocateFrame, Field::DeallocateFrame};
    }
    Kind kind;
    constexpr Field(Kind kind) : kind(kind) {}
@@ -638,6 +670,8 @@ struct Allocator {
        return IGM.Int32Ty;
      case Field::Allocate:
      case Field::Deallocate:
+      case Field::AllocateFrame:
+      case Field::DeallocateFrame:
        return IGM.PtrTy;
      }
    }
@@ -648,6 +682,8 @@ struct Allocator {
        return Alignment(4);
      case Field::Allocate:
      case Field::Deallocate:
+      case Field::AllocateFrame:
+      case Field::DeallocateFrame:
        return IGM.getPointerAlignment();
      }
    }
@@ -660,6 +696,10 @@ struct Allocator {
        return "allocate_fn";
      case Field::Deallocate:
        return "deallocate_fn";
+      case Field::AllocateFrame:
+        return "allocate_frame_fn";
+      case Field::DeallocateFrame:
+        return "deallocate_frame_fn";
      }
    }

@@ -669,6 +709,8 @@ struct Allocator {
        return false;
      case Field::Allocate:
      case Field::Deallocate:
+      case Field::AllocateFrame:
+      case Field::DeallocateFrame:
        return true;
      }
    }
@@ -678,8 +720,10 @@ struct Allocator {
      case Flags:
        llvm_unreachable("not a function");
      case Field::Allocate:
+      case Field::AllocateFrame:
        return IGM.CoroAllocateFnTy;
      case Field::Deallocate:
+      case Field::DeallocateFrame:
        return IGM.CoroDeallocateFnTy;
      }
    }
@@ -690,6 +734,8 @@ struct Allocator {
        llvm_unreachable("not a function");
      case Field::Allocate:
      case Field::Deallocate:
+      case Field::AllocateFrame:
+      case Field::DeallocateFrame:
        auto attrs = llvm::AttributeList();
        IGM.addSwiftCoroAttributes(attrs, 0);
        return attrs;
@@ -704,6 +750,10 @@ struct Allocator {
        return IGM.getOptions().PointerAuth.CoroAllocationFunction;
      case Field::Deallocate:
        return IGM.getOptions().PointerAuth.CoroDeallocationFunction;
+      case Field::AllocateFrame:
+        return IGM.getOptions().PointerAuth.CoroFrameAllocationFunction;
+      case Field::DeallocateFrame:
+        return IGM.getOptions().PointerAuth.CoroFrameDeallocationFunction;
      }
    }

@@ -715,6 +765,10 @@ struct Allocator {
        return PointerAuthEntity::Special::CoroAllocationFunction;
      case Field::Deallocate:
        return PointerAuthEntity::Special::CoroDeallocationFunction;
+      case Field::AllocateFrame:
+        return PointerAuthEntity::Special::CoroFrameAllocationFunction;
+      case Field::DeallocateFrame:
+        return PointerAuthEntity::Special::CoroFrameDeallocationFunction;
      }
    }
  };
@@ -759,10 +813,22 @@ struct Allocator {

  llvm::Value *getFlags() { return getField(Field::Flags); }

-  FunctionPointer getAllocate() { return getFunctionPointer(Field::Allocate); }
+  FunctionPointer getAllocate(AllocationKind kind) {
+    switch (kind) {
+    case AllocationKind::Normal:
+      return getAllocate();
+    case AllocationKind::Frame:
+      return getAllocateFrame();
+    }
+  }

-  FunctionPointer getDeallocate() {
-    return getFunctionPointer(Field::Deallocate);
+  FunctionPointer getDeallocate(AllocationKind kind) {
+    switch (kind) {
+    case AllocationKind::Normal:
+      return getDeallocate();
+    case AllocationKind::Frame:
+      return getDeallocateFrame();
+    }
  }

  void emitOnNullAllocator(StringRef nullBlockName, StringRef nonnullBlockName,
@@ -781,6 +847,20 @@ struct Allocator {
  }

 private:
+  FunctionPointer getAllocate() { return getFunctionPointer(Field::Allocate); }
+
+  FunctionPointer getDeallocate() {
+    return getFunctionPointer(Field::Deallocate);
+  }
+
+  FunctionPointer getAllocateFrame() {
+    return getFunctionPointer(Field::AllocateFrame);
+  }
+
+  FunctionPointer getDeallocateFrame() {
+    return getFunctionPointer(Field::DeallocateFrame);
+  }
+
  FunctionPointer getFunctionPointer(Field field) {
    llvm::Value *callee = getField(field);
    if (auto &schema = field.getSchema(IGF.IGM)) {
@@ -794,13 +874,14 @@ private:
                  field.getFunctionAttributeList(IGF.IGM), IGF.IGM.SwiftCC));
  }
 };
-} // end anonymous namespace

-llvm::Constant *swift::irgen::getCoroAllocFn(IRGenModule &IGM) {
-  auto isSwiftCoroCCAvailable = IGM.SwiftCoroCC == llvm::CallingConv::SwiftCoro;
+llvm::Constant *getCoroAllocFn(AllocationKind kind, IRGenModule &IGM) {
  return IGM.getOrCreateHelperFunction(
-      "_swift_coro_alloc", IGM.Int8PtrTy, {IGM.CoroAllocatorPtrTy, IGM.SizeTy},
-      [isSwiftCoroCCAvailable](IRGenFunction &IGF) {
+      kind.getAllocationFunctionName(), IGM.Int8PtrTy,
+      {IGM.CoroAllocatorPtrTy, IGM.SizeTy},
+      [kind](IRGenFunction &IGF) {
+        auto isSwiftCoroCCAvailable =
+            IGF.IGM.SwiftCoroCC == llvm::CallingConv::SwiftCoro;
        auto parameters = IGF.collectParameters();
        auto *allocatorValue = parameters.claimNext();
        auto allocator = Allocator(allocatorValue, IGF);
@@ -821,7 +902,7 @@ llvm::Constant *swift::irgen::getCoroAllocFn(IRGenModule &IGM) {
            IGF.Builder.CreateRet(alloca);
          });
        }
-        auto fnPtr = allocator.getAllocate();
+        auto fnPtr = allocator.getAllocate(kind);
        auto *call = IGF.Builder.CreateCall(fnPtr, {allocatorValue, size});
        call->setDoesNotThrow();
        call->setCallingConv(IGF.IGM.SwiftCC);
@@ -837,12 +918,13 @@ llvm::Constant *swift::irgen::getCoroAllocFn(IRGenModule &IGM) {
      });
 }

-llvm::Constant *swift::irgen::getCoroDeallocFn(IRGenModule &IGM) {
-  auto isSwiftCoroCCAvailable = IGM.SwiftCoroCC == llvm::CallingConv::SwiftCoro;
+llvm::Constant *getCoroDeallocFn(AllocationKind kind, IRGenModule &IGM) {
  return IGM.getOrCreateHelperFunction(
-      "_swift_coro_dealloc", IGM.VoidTy,
+      kind.getDeallocationFunctionName(), IGM.VoidTy,
      {IGM.CoroAllocatorPtrTy, IGM.Int8PtrTy},
-      [isSwiftCoroCCAvailable](IRGenFunction &IGF) {
+      [kind](IRGenFunction &IGF) {
+        auto isSwiftCoroCCAvailable =
+            IGF.IGM.SwiftCoroCC == llvm::CallingConv::SwiftCoro;
        auto parameters = IGF.collectParameters();
        auto *allocatorValue = parameters.claimNext();
        auto allocator = Allocator(allocatorValue, IGF);
@@ -876,7 +958,7 @@ llvm::Constant *swift::irgen::getCoroDeallocFn(IRGenModule &IGM) {
        IGF.Builder.CreateRetVoid();
        // Start emitting the "normal" block.
        IGF.Builder.emitBlock(normalBlock);
-        auto fnPtr = allocator.getDeallocate();
+        auto fnPtr = allocator.getDeallocate(kind);
        auto *call = IGF.Builder.CreateCall(fnPtr, {allocatorValue, ptr});
        call->setDoesNotThrow();
        call->setCallingConv(IGF.IGM.SwiftCC);
@@ -892,10 +974,29 @@ llvm::Constant *swift::irgen::getCoroDeallocFn(IRGenModule &IGM) {
      });
 }

+} // end anonymous namespace
+
+llvm::Constant *swift::irgen::getCoroAllocFn(IRGenModule &IGM) {
+  return getCoroAllocFn(AllocationKind::Normal, IGM);
+}
+
+llvm::Constant *swift::irgen::getCoroAllocFrameFn(IRGenModule &IGM) {
+  return getCoroAllocFn(AllocationKind::Frame, IGM);
+}
+
+llvm::Constant *swift::irgen::getCoroDeallocFn(IRGenModule &IGM) {
+  return getCoroDeallocFn(AllocationKind::Normal, IGM);
+}
+
+llvm::Constant *swift::irgen::getCoroDeallocFrameFn(IRGenModule &IGM) {
+  return getCoroDeallocFn(AllocationKind::Frame, IGM);
+}
+
 using GetAllocFunctionPointer = FunctionPointer (IRGenModule::*)();

 static llvm::Constant *
-getAddrOfSwiftCoroAllocThunk(StringRef name, GetAllocFunctionPointer getAlloc,
+getAddrOfSwiftCoroAllocThunk(StringRef name,
+                             GetAllocFunctionPointer getAlloc,
                             IRGenModule &IGM) {
  auto *ty = IGM.CoroAllocateFnTy;
  return IGM.getOrCreateHelperFunction(
@@ -920,8 +1021,10 @@ getAddrOfSwiftCoroAllocThunk(StringRef name, GetAllocFunctionPointer getAlloc,

 using GetDeallocFunctionPointer = FunctionPointer (IRGenModule::*)();

-static llvm::Constant *getAddrOfSwiftCoroDeallocThunk(
-    StringRef name, GetDeallocFunctionPointer getDealloc, IRGenModule &IGM) {
+static llvm::Constant *
+getAddrOfSwiftCoroDeallocThunk(StringRef name,
+                               GetDeallocFunctionPointer getDealloc,
+                               IRGenModule &IGM) {
  auto *ty = IGM.CoroDeallocateFnTy;
  return IGM.getOrCreateHelperFunction(
      name, ty->getReturnType(), ty->params(),
@@ -945,7 +1048,8 @@ static llvm::Constant *getAddrOfSwiftCoroDeallocThunk(

 static llvm::Constant *getAddrOfGlobalCoroAllocator(
    IRGenModule &IGM, CoroAllocatorKind kind, bool shouldDeallocateImmediately,
-    llvm::Constant *allocFn, llvm::Constant *deallocFn) {
+    llvm::Constant *allocFn, llvm::Constant *deallocFn,
+    llvm::Constant *allocFrameFn, llvm::Constant *deallocFrameFn) {
  auto entity = LinkEntity::forCoroAllocator(kind);
  auto taskAllocator = IGM.getOrCreateLazyGlobalVariable(
      entity,
@@ -964,6 +1068,10 @@ static llvm::Constant *getAddrOfGlobalCoroAllocator(
                return allocFn;
              case Allocator::Field::Deallocate:
                return deallocFn;
+              case Allocator::Field::AllocateFrame:
+                return allocFrameFn;
+              case Allocator::Field::DeallocateFrame:
+                return deallocFrameFn;
              }
            });
      },
@@ -971,39 +1079,52 @@ static llvm::Constant *getAddrOfGlobalCoroAllocator(
  return taskAllocator;
 }

-static llvm::Constant *getAddrOfSwiftCoroMalloc(IRGenModule &IGM) {
-  return getAddrOfSwiftCoroAllocThunk(
-      "_swift_coro_malloc", &IRGenModule::getMallocFunctionPointer, IGM);
+static llvm::Constant *getAddrOfSwiftCoroMalloc(
+                                                IRGenModule &IGM) {
+  return getAddrOfSwiftCoroAllocThunk("_swift_coro_malloc",
+                                      &IRGenModule::getMallocFunctionPointer,
+                                      IGM);
 }

-static llvm::Constant *getAddrOfSwiftCoroFree(IRGenModule &IGM) {
-  return getAddrOfSwiftCoroDeallocThunk(
-      "_swift_coro_free", &IRGenModule::getFreeFunctionPointer, IGM);
+static llvm::Constant *getAddrOfSwiftCoroFree(
+                                              IRGenModule &IGM) {
+  return getAddrOfSwiftCoroDeallocThunk("_swift_coro_free",
+                                        &IRGenModule::getFreeFunctionPointer,
+                                        IGM);
 }

 llvm::Constant *IRGenModule::getAddrOfGlobalCoroMallocAllocator() {
-  return getAddrOfGlobalCoroAllocator(*this, CoroAllocatorKind::Malloc,
-                                      /*shouldDeallocateImmediately=*/true,
-                                      getAddrOfSwiftCoroMalloc(*this),
-                                      getAddrOfSwiftCoroFree(*this));
+  return getAddrOfGlobalCoroAllocator(
+      *this, CoroAllocatorKind::Malloc,
+      /*shouldDeallocateImmediately=*/true,
+      getAddrOfSwiftCoroMalloc(*this),
+      getAddrOfSwiftCoroFree(*this),
+      getAddrOfSwiftCoroMalloc(*this),
+      getAddrOfSwiftCoroFree(*this));
 }

-static llvm::Constant *getAddrOfSwiftCoroTaskAlloc(IRGenModule &IGM) {
-  return getAddrOfSwiftCoroAllocThunk(
-      "_swift_coro_task_alloc", &IRGenModule::getTaskAllocFunctionPointer, IGM);
+static llvm::Constant *
+getAddrOfSwiftCoroTaskAlloc(IRGenModule &IGM) {
+  return getAddrOfSwiftCoroAllocThunk("_swift_coro_task_alloc",
+                                      &IRGenModule::getTaskAllocFunctionPointer,
+                                      IGM);
 }

-static llvm::Constant *getAddrOfSwiftCoroTaskDealloc(IRGenModule &IGM) {
+static llvm::Constant *
+getAddrOfSwiftCoroTaskDealloc(IRGenModule &IGM) {
  return getAddrOfSwiftCoroDeallocThunk(
-      "_swift_coro_task_dealloc", &IRGenModule::getTaskDeallocFunctionPointer,
-      IGM);
+      "_swift_coro_task_dealloc",
+      &IRGenModule::getTaskDeallocFunctionPointer, IGM);
 }

 llvm::Constant *IRGenModule::getAddrOfGlobalCoroAsyncTaskAllocator() {
-  return getAddrOfGlobalCoroAllocator(*this, CoroAllocatorKind::Async,
-                                      /*shouldDeallocateImmediately=*/false,
-                                      getAddrOfSwiftCoroTaskAlloc(*this),
-                                      getAddrOfSwiftCoroTaskDealloc(*this));
+  return getAddrOfGlobalCoroAllocator(
+      *this, CoroAllocatorKind::Async,
+      /*shouldDeallocateImmediately=*/false,
+      getAddrOfSwiftCoroTaskAlloc(*this),
+      getAddrOfSwiftCoroTaskDealloc(*this),
+      getAddrOfSwiftCoroTaskAlloc(*this),
+      getAddrOfSwiftCoroTaskDealloc(*this));
 }

 llvm::Value *
--- a/lib/IRGen/GenCoro.h
+++ b/lib/IRGen/GenCoro.h
@@ -16,7 +16,9 @@ namespace irgen {
 class IRGenModule;

 llvm::Constant *getCoroAllocFn(IRGenModule &IGM);
+llvm::Constant *getCoroAllocFrameFn(IRGenModule &IGM);
 llvm::Constant *getCoroDeallocFn(IRGenModule &IGM);
+llvm::Constant *getCoroDeallocFrameFn(IRGenModule &IGM);

 llvm::Value *
 emitYieldOnce2CoroutineAllocator(IRGenFunction &IGF,
--- a/lib/IRGen/GenOpaque.cpp
+++ b/lib/IRGen/GenOpaque.cpp
@@ -560,14 +560,14 @@ StackAddress IRGenFunction::emitDynamicAlloca(SILType T,
                                              const llvm::Twine &name) {
  llvm::Value *size = emitLoadOfSize(*this, T);
  return emitDynamicAlloca(IGM.Int8Ty, size, Alignment(16), AllowsTaskAlloc,
-                           name);
+                           IsNotForCalleeCoroutineFrame, name);
 }

-StackAddress IRGenFunction::emitDynamicAlloca(llvm::Type *eltTy,
-                                              llvm::Value *arraySize,
-                                              Alignment align,
-                                              AllowsTaskAlloc_t allowTaskAlloc,
-                                              const llvm::Twine &name) {
+StackAddress IRGenFunction::emitDynamicAlloca(
+    llvm::Type *eltTy, llvm::Value *arraySize, Alignment align,
+    AllowsTaskAlloc_t allowTaskAlloc,
+    IsForCalleeCoroutineFrame_t forCalleeCoroutineFrame,
+    const llvm::Twine &name) {
  // Async functions call task alloc.
  if (allowTaskAlloc && isAsync()) {
    llvm::Value *byteCount;
@@ -601,9 +601,10 @@ StackAddress IRGenFunction::emitDynamicAlloca(llvm::Type *eltTy,
    auto alignment = llvm::ConstantInt::get(IGM.Int32Ty, align.getValue());

    // Allocate memory.  This produces an abstract token.
-    auto allocToken =
-        Builder.CreateIntrinsicCall(llvm::Intrinsic::coro_alloca_alloc,
-                                    {IGM.SizeTy}, {byteCount, alignment});
+    auto *allocToken = Builder.CreateIntrinsicCall(
+        forCalleeCoroutineFrame ? llvm::Intrinsic::coro_alloca_alloc_frame
+                                : llvm::Intrinsic::coro_alloca_alloc,
+        {IGM.SizeTy}, {byteCount, alignment});

    // Get the allocation result.
    auto ptr = Builder.CreateIntrinsicCall(llvm::Intrinsic::coro_alloca_get,
@@ -643,7 +644,8 @@ StackAddress IRGenFunction::emitDynamicAlloca(llvm::Type *eltTy,
 /// location before the dynamic alloca's call.
 void IRGenFunction::emitDeallocateDynamicAlloca(StackAddress address,
                                                bool allowTaskDealloc,
-                                                bool useTaskDeallocThrough) {
+                                                bool useTaskDeallocThrough,
+                                                bool forCalleeCoroutineFrame) {
  // Async function use taskDealloc.
  if (allowTaskDealloc && isAsync() && address.getAddress().isValid()) {
    if (useTaskDeallocThrough) {
@@ -671,7 +673,10 @@ void IRGenFunction::emitDeallocateDynamicAlloca(StackAddress address,
 #endif
      return;
    }
-    Builder.CreateIntrinsicCall(llvm::Intrinsic::coro_alloca_free, allocToken);
+    Builder.CreateIntrinsicCall(forCalleeCoroutineFrame
+                                    ? llvm::Intrinsic::coro_alloca_free_frame
+                                    : llvm::Intrinsic::coro_alloca_free,
+                                allocToken);
    return;
  }
  // Otherwise, call llvm.stackrestore if an address was saved.
--- a/lib/IRGen/GenPointerAuth.cpp
+++ b/lib/IRGen/GenPointerAuth.cpp
@@ -399,6 +399,10 @@ PointerAuthEntity::getDeclDiscriminator(IRGenModule &IGM) const {
        return SpecialPointerAuthDiscriminators::CoroAllocationFunction;
      case Special::CoroDeallocationFunction:
        return SpecialPointerAuthDiscriminators::CoroDeallocationFunction;
+      case Special::CoroFrameAllocationFunction:
+        return SpecialPointerAuthDiscriminators::CoroFrameAllocationFunction;
+      case Special::CoroFrameDeallocationFunction:
+        return SpecialPointerAuthDiscriminators::CoroFrameDeallocationFunction;
      }
      llvm_unreachable("bad kind");
    };
--- a/lib/IRGen/GenPointerAuth.h
+++ b/lib/IRGen/GenPointerAuth.h
@@ -70,6 +70,8 @@ public:
    TypeLayoutString,
    CoroAllocationFunction,
    CoroDeallocationFunction,
+    CoroFrameAllocationFunction,
+    CoroFrameDeallocationFunction,
  };

 private:
--- a/lib/IRGen/IRGen.cpp
+++ b/lib/IRGen/IRGen.cpp
@@ -1098,6 +1098,14 @@ static void setPointerAuthOptions(PointerAuthOptions &opts,
  opts.CoroDeallocationFunction = PointerAuthSchema(
      codeKey, /*address*/ false, Discrimination::Constant,
      SpecialPointerAuthDiscriminators::CoroDeallocationFunction);
+
+  opts.CoroAllocationFunction = PointerAuthSchema(
+      codeKey, /*address*/ false, Discrimination::Constant,
+      SpecialPointerAuthDiscriminators::CoroFrameAllocationFunction);
+
+  opts.CoroDeallocationFunction = PointerAuthSchema(
+      codeKey, /*address*/ false, Discrimination::Constant,
+      SpecialPointerAuthDiscriminators::CoroFrameDeallocationFunction);
 }

 std::unique_ptr<llvm::TargetMachine>
--- a/lib/IRGen/IRGenFunction.h
+++ b/lib/IRGen/IRGenFunction.h
@@ -71,6 +71,11 @@ enum AllowsTaskAlloc_t : bool {
  AllowsTaskAlloc = true,
 };

+enum IsForCalleeCoroutineFrame_t : bool {
+  IsNotForCalleeCoroutineFrame,
+  IsForCalleeCoroutineFrame,
+};
+
 /// IRGenFunction - Primary class for emitting LLVM instructions for a
 /// specific function.
 class IRGenFunction {
@@ -310,10 +315,13 @@ public:
  StackAddress
  emitDynamicAlloca(llvm::Type *eltTy, llvm::Value *arraySize, Alignment align,
                    AllowsTaskAlloc_t allowTaskAlloc = AllowsTaskAlloc,
+                    IsForCalleeCoroutineFrame_t forCalleeCoroutineFrame =
+                        IsNotForCalleeCoroutineFrame,
                    const llvm::Twine &name = "");
  void emitDeallocateDynamicAlloca(StackAddress address,
                                   bool allowTaskDealloc = true,
-                                   bool useTaskDeallocThrough = false);
+                                   bool useTaskDeallocThrough = false,
+                                   bool forCalleeCoroutineFrame = false);

  llvm::BasicBlock *createBasicBlock(const llvm::Twine &Name);
  const TypeInfo &getTypeInfoForUnlowered(Type subst);
--- a/lib/IRGen/IRGenModule.cpp
+++ b/lib/IRGen/IRGenModule.cpp
@@ -783,6 +783,8 @@ IRGenModule::IRGenModule(IRGenerator &irgen,
                                         Int32Ty, // CoroAllocator.Flags
                                         PtrTy, // allocate
                                         PtrTy, // deallocate
+                                         PtrTy, // allocateFrame
+                                         PtrTy, // deallocateFrame
                                     });
  SwiftImplicitActorType =
      createStructType(*this, "swift.implicit_isolated_actor_type",
--- a/lib/IRGen/IRGenSIL.cpp
+++ b/lib/IRGen/IRGenSIL.cpp
@@ -3721,8 +3721,9 @@ static void emitBuiltinStackAlloc(IRGenSILFunction &IGF,
    return;
  }

-  auto stackAddress = IGF.emitDynamicAlloca(
-      IGF.IGM.Int8Ty, size, align, DoesNotAllowTaskAlloc, "temp_alloc");
+  auto stackAddress =
+      IGF.emitDynamicAlloca(IGF.IGM.Int8Ty, size, align, DoesNotAllowTaskAlloc,
+                            IsNotForCalleeCoroutineFrame, "temp_alloc");
  IGF.setLoweredStackAddress(i, stackAddress);
 }

--- a/test/IRGen/coroutine_accessors.swift
+++ b/test/IRGen/coroutine_accessors.swift
@@ -308,7 +308,7 @@ public var force_yield_once_convention : () {
 // CHECK-SAME:        _swift_coro_malloc_allocator
 // CHECK-SAME:    )
 // CHECK:         call void @llvm.lifetime.end.p0(i64 -1, ptr [[FRAME]])
-// CHECK:         call void @llvm.coro.alloca.free(token [[ALLOCATION]])
+// CHECK:         call void @llvm.coro.alloca.free.frame(token [[ALLOCATION]])
 // CHECK:       }
  @_silgen_name("increment_irm_yield_once")
  _modify {
@@ -338,8 +338,8 @@ public var force_yield_once_2_convention : () {
 //           :            i32 1
 //           :        )
 // CHECK-64:      [[SIZE_64:%[^,]+]] = zext i32 {{%[^,]+}} to i64
-// CHECK-64:      [[ALLOCATION:%[^,]+]] = call token @llvm.coro.alloca.alloc.i64(i64 [[SIZE_64]], i32 16)
-// CHECK-32:      [[ALLOCATION:%[^,]+]] = call token @llvm.coro.alloca.alloc.i32(i32 {{%[^,]+}}, i32 16)
+// CHECK-64:      [[ALLOCATION:%[^,]+]] = call token @llvm.coro.alloca.alloc.frame.i64(i64 [[SIZE_64]], i32 16)
+// CHECK-32:      [[ALLOCATION:%[^,]+]] = call token @llvm.coro.alloca.alloc.frame.i32(i32 {{%[^,]+}}, i32 16)
 // CHECK:         [[FRAME:%[^,]+]] = call ptr @llvm.coro.alloca.get(token [[ALLOCATION]])
 // CHECK:         call void @llvm.lifetime.start.p0(i64 -1, ptr [[FRAME]])
 // CHECK:         [[RAMP:%[^,]+]] = call ptr @llvm.coro.prepare.retcon(ptr @"$s19coroutine_accessors1SV3irmSivx")
@@ -357,7 +357,7 @@ public var force_yield_once_2_convention : () {
 // CHECK-SAME:        [[ALLOCATOR]]
 // CHECK-SAME:    )
 // CHECK:         call void @llvm.lifetime.end.p0(i64 -1, ptr [[FRAME]])
-// CHECK:         call void @llvm.coro.alloca.free(token [[ALLOCATION]])
+// CHECK:         call void @llvm.coro.alloca.free.frame(token [[ALLOCATION]])
 // CHECK:       }
  @_silgen_name("increment_irm_yield_once_2")
  modify {
--- a/test/IRGen/coroutine_accessors_popless.swift
+++ b/test/IRGen/coroutine_accessors_popless.swift
@@ -255,7 +255,7 @@ public var force_yield_once_convention : () {
 // CHECK-SAME:        _swift_coro_malloc_allocator
 // CHECK-SAME:    )
 // CHECK:         call void @llvm.lifetime.end.p0(i64 -1, ptr [[FRAME]])
-// CHECK:         call void @llvm.coro.alloca.free(token [[ALLOCATION]])
+// CHECK:         call void @llvm.coro.alloca.free.frame(token [[ALLOCATION]])
 // CHECK:       }
  @_silgen_name("increment_i_yield_once")
  _modify {
@@ -284,7 +284,7 @@ public var force_yield_once_2_convention : () {
 //           :            i32 1
 //           :        )
 // CHECK:         [[SIZE:%[^,]+]] = zext i32 {{%[^,]+}} to i64
-// CHECK:         [[ALLOCATION:%[^,]+]] = call token @llvm.coro.alloca.alloc.i64(i64 [[SIZE]], i32 16)
+// CHECK:         [[ALLOCATION:%[^,]+]] = call token @llvm.coro.alloca.alloc.frame.i64(i64 [[SIZE]], i32 16)
 // CHECK:         [[FRAME:%[^,]+]] = call ptr @llvm.coro.alloca.get(token [[ALLOCATION]])
 // CHECK:         call void @llvm.lifetime.start.p0(i64 -1, ptr [[FRAME]])
 // CHECK:         [[RAMP:%[^,]+]] = call ptr @llvm.coro.prepare.retcon(ptr @"$s27coroutine_accessors_popless1iSivx")
@@ -302,7 +302,7 @@ public var force_yield_once_2_convention : () {
 // CHECK-SAME:        [[ALLOCATOR]]
 // CHECK-SAME:    )
 // CHECK:         call void @llvm.lifetime.end.p0(i64 -1, ptr [[FRAME]])
-// CHECK:         call void @llvm.coro.alloca.free(token [[ALLOCATION]])
+// CHECK:         call void @llvm.coro.alloca.free.frame(token [[ALLOCATION]])
 // CHECK:       }
  @_silgen_name("increment_i_yield_once_2")
  modify {