averello/swift-mirror
1
0
Fork 0
mirror of https://github.com/apple/swift.git synced 2025-12-14 20:36:38 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #85055 from nate-chandler/general-coro/20251016/1

Browse Source 
[CoroutineAccessors] Provide frame to de/allocation functions.
This commit is contained in:
nate-chandler
2025-10-23 20:17:52 -07:00
committed by GitHub
parent 25543a56fc cdf9f9d58b
commit 9069d32078
16 changed files with 446 additions and 155 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
include/swift/ABI/Coro.h
View File

@@ -56,14 +56,20 @@ public:
setShouldDeallocateImmediately)
};
struct CoroAllocator;
using CoroAllocation = void *;
using CoroAllocateFn = CoroAllocation (*)(size_t);
using CoroDealllocateFn = void (*)(CoroAllocation);
using CoroAllocateFn = CoroAllocation (*)(CoroAllocator *, size_t);
using CoroDealllocateFn = void (*)(CoroAllocator *, CoroAllocation);
using CoroAllocateFrameFn = CoroAllocation (*)(CoroAllocator *, size_t);
using CoroDealllocateFrameFn = void (*)(CoroAllocator *, CoroAllocation);
struct CoroAllocator {
CoroAllocatorFlags flags;
CoroAllocateFn allocate;
CoroDealllocateFn deallocate;
CoroAllocateFrameFn allocateFrame;
CoroDealllocateFrameFn deallocateFrame;
/// Whether the allocator should deallocate memory on calls to
/// swift_coro_dealloc.

2
include/swift/ABI/MetadataValues.h
View File

@@ -1797,6 +1797,8 @@ namespace SpecialPointerAuthDiscriminators {
/// Function pointers stored in the coro allocator struct.
const uint16_t CoroAllocationFunction = 0x5f95; // = 24469
const uint16_t CoroDeallocationFunction = 0x9faf; // = 40879
const uint16_t CoroFrameAllocationFunction = 0xd251; // = 53841
const uint16_t CoroFrameDeallocationFunction = 0x5ba8; // = 23464
}
/// The number of arguments that will be passed directly to a generic

8
include/swift/AST/IRGenOptions.h
View File

@@ -257,6 +257,14 @@ struct PointerAuthOptions : clang::PointerAuthOptions {
/// Stored in a coro allocator struct, the function used to deallocate memory.
PointerAuthSchema CoroDeallocationFunction;
/// Stored in a coro allocator struct, the function used to allocate a callee
/// coroutine's fixed-size frame.
PointerAuthSchema CoroFrameAllocationFunction;
/// Stored in a coro allocator struct, the function used to deallocate a
/// callee coroutine's fixed-size frame.
PointerAuthSchema CoroFrameDeallocationFunction;
};
enum class JITDebugArtifact : unsigned {

14
lib/IRGen/GenCall.cpp
View File

@@ -5177,10 +5177,12 @@ void irgen::emitYieldOnce2CoroutineEntry(IRGenFunction &IGF,
IGF.setCoroutineAllocator(allocator);
auto allocFn = IGF.IGM.getOpaquePtr(getCoroAllocFn(IGF.IGM));
auto deallocFn = IGF.IGM.getOpaquePtr(getCoroDeallocFn(IGF.IGM));
auto allocFrameFn = IGF.IGM.getOpaquePtr(getCoroAllocFrameFn(IGF.IGM));
auto deallocFrameFn = IGF.IGM.getOpaquePtr(getCoroDeallocFrameFn(IGF.IGM));
emitRetconCoroutineEntry(
IGF, fnType, buffer, llvm::Intrinsic::coro_id_retcon_once_dynamic,
Size(-1) /*dynamic-to-IRGen size*/, IGF.IGM.getCoroStaticFrameAlignment(),
{cfp, allocator}, allocFn, deallocFn, {});
{cfp, allocator}, allocFn, deallocFn, {allocFrameFn, deallocFrameFn});
}
void irgen::emitYieldOnce2CoroutineEntry(
IRGenFunction &IGF, LinkEntity coroFunction, CanSILFunctionType fnType,
@@ -5266,9 +5268,10 @@ StackAddress irgen::emitAllocCoroStaticFrame(IRGenFunction &IGF,
// TODO: Avoid swift_task_alloc (async) and malloc (yield_once) if the
// suspension doesn't span an apply of an async function or a yield
// respectively.
auto retval =
IGF.emitDynamicAlloca(IGF.IGM.Int8Ty, size, Alignment(MaximumAlignment),
/*allowTaskAlloc*/ true, "caller-coro-frame");
auto retval = IGF.emitDynamicAlloca(
IGF.IGM.Int8Ty, size, Alignment(MaximumAlignment), AllowsTaskAlloc,
IsForCalleeCoroutineFrame_t(IGF.isCalleeAllocatedCoroutine()),
"callee-coro-frame");
IGF.Builder.CreateLifetimeStart(retval.getAddress(),
Size(-1) /*dynamic size*/);
return retval;
@@ -5276,7 +5279,8 @@ StackAddress irgen::emitAllocCoroStaticFrame(IRGenFunction &IGF,
void irgen::emitDeallocCoroStaticFrame(IRGenFunction &IGF, StackAddress frame) {
IGF.Builder.CreateLifetimeEnd(frame.getAddress(), Size(-1) /*dynamic size*/);
IGF.emitDeallocateDynamicAlloca(frame, /*allowTaskAlloc*/ true,
/*useTaskDeallocThrough*/ true);
/*useTaskDeallocThrough*/ true,
/*forCalleeCoroutineFrame*/ true);
}
llvm::Value *irgen::emitYield(IRGenFunction &IGF,

376
lib/IRGen/GenCoro.cpp
View File

@@ -24,6 +24,7 @@
#include "GenPointerAuth.h"
#include "IRGenFunction.h"
#include "IRGenModule.h"
#include "clang/CodeGen/CodeGenABITypes.h"
using namespace swift;
using namespace irgen;
@@ -618,16 +619,50 @@ void IRGenFunction::emitTaskDeallocThrough(Address address) {
}
namespace {
struct AllocationKind {
enum Kind {
Normal,
Frame,
};
Kind kind;
constexpr AllocationKind(Kind kind) : kind(kind) {}
constexpr operator Kind() { return kind; }
StringRef getAllocationFunctionName() {
switch (kind) {
case Normal:
return "_swift_coro_alloc";
case Frame:
return "_swift_coro_alloc_frame";
}
}
StringRef getDeallocationFunctionName() {
switch (kind) {
case Normal:
return "_swift_coro_dealloc";
case Frame:
return "_swift_coro_dealloc_frame";
}
}
};
struct Allocator {
struct Field {
enum Kind : uint8_t {
Flags = 0,
Allocate = 1,
Deallocate = 2,
AllocateFrame = 3,
DeallocateFrame = 4,
};
constexpr static std::array<Field, 5> allFields() {
return {Field::Flags, Field::Allocate, Field::Deallocate,
Field::AllocateFrame, Field::DeallocateFrame};
}
Kind kind;
Field(Kind kind) : kind(kind) {}
operator Kind() { return kind; }
constexpr Field(Kind kind) : kind(kind) {}
constexpr operator Kind() { return kind; }
llvm::Type *getType(IRGenModule &IGM) {
switch (kind) {
@@ -635,6 +670,8 @@ struct Allocator {
return IGM.Int32Ty;
case Field::Allocate:
case Field::Deallocate:
case Field::AllocateFrame:
case Field::DeallocateFrame:
return IGM.PtrTy;
}
}
@@ -645,6 +682,8 @@ struct Allocator {
return Alignment(4);
case Field::Allocate:
case Field::Deallocate:
case Field::AllocateFrame:
case Field::DeallocateFrame:
return IGM.getPointerAlignment();
}
}
@@ -657,6 +696,22 @@ struct Allocator {
return "allocate_fn";
case Field::Deallocate:
return "deallocate_fn";
case Field::AllocateFrame:
return "allocate_frame_fn";
case Field::DeallocateFrame:
return "deallocate_frame_fn";
}
}
bool isFunctionPointer() {
switch (kind) {
case Flags:
return false;
case Field::Allocate:
case Field::Deallocate:
case Field::AllocateFrame:
case Field::DeallocateFrame:
return true;
}
}
@@ -665,12 +720,28 @@ struct Allocator {
case Flags:
llvm_unreachable("not a function");
case Field::Allocate:
case Field::AllocateFrame:
return IGM.CoroAllocateFnTy;
case Field::Deallocate:
case Field::DeallocateFrame:
return IGM.CoroDeallocateFnTy;
}
}
llvm::AttributeList getFunctionAttributeList(IRGenModule &IGM) {
switch (kind) {
case Flags:
llvm_unreachable("not a function");
case Field::Allocate:
case Field::Deallocate:
case Field::AllocateFrame:
case Field::DeallocateFrame:
auto attrs = llvm::AttributeList();
IGM.addSwiftCoroAttributes(attrs, 1);
return attrs;
}
}
const PointerAuthSchema &getSchema(IRGenModule &IGM) {
switch (kind) {
case Flags:
@@ -679,10 +750,51 @@ struct Allocator {
return IGM.getOptions().PointerAuth.CoroAllocationFunction;
case Field::Deallocate:
return IGM.getOptions().PointerAuth.CoroDeallocationFunction;
case Field::AllocateFrame:
return IGM.getOptions().PointerAuth.CoroFrameAllocationFunction;
case Field::DeallocateFrame:
return IGM.getOptions().PointerAuth.CoroFrameDeallocationFunction;
}
}
const PointerAuthEntity getEntity(IRGenModule &IGM) {
switch (kind) {
case Flags:
llvm_unreachable("no entity");
case Field::Allocate:
return PointerAuthEntity::Special::CoroAllocationFunction;
case Field::Deallocate:
return PointerAuthEntity::Special::CoroDeallocationFunction;
case Field::AllocateFrame:
return PointerAuthEntity::Special::CoroFrameAllocationFunction;
case Field::DeallocateFrame:
return PointerAuthEntity::Special::CoroFrameDeallocationFunction;
}
}
};
static void visitFields(llvm::function_ref<void(Field)> visitor) {
for (auto field : Field::allFields()) {
visitor(field);
}
}
static ConstantInitFuture
buildStruct(ConstantInitBuilder &builder, IRGenModule &IGM,
llvm::function_ref<llvm::Constant *(Field)> valueForField) {
auto allocator = builder.beginStruct(IGM.CoroAllocatorTy);
visitFields([&](auto field) {
auto *value = valueForField(field);
if (!field.isFunctionPointer()) {
allocator.add(value);
return;
}
allocator.addSignedPointer(value, field.getSchema(IGM),
field.getEntity(IGM));
});
return allocator.finishAndCreateFuture();
}
llvm::Value *address;
IRGenFunction &IGF;
@@ -701,10 +813,22 @@ struct Allocator {
llvm::Value *getFlags() { return getField(Field::Flags); }
FunctionPointer getAllocate() { return getFunctionPointer(Field::Allocate); }
FunctionPointer getAllocate(AllocationKind kind) {
switch (kind) {
case AllocationKind::Normal:
return getAllocate();
case AllocationKind::Frame:
return getAllocateFrame();
}
}
FunctionPointer getDeallocate() {
return getFunctionPointer(Field::Deallocate);
FunctionPointer getDeallocate(AllocationKind kind) {
switch (kind) {
case AllocationKind::Normal:
return getDeallocate();
case AllocationKind::Frame:
return getDeallocateFrame();
}
}
void emitOnNullAllocator(StringRef nullBlockName, StringRef nonnullBlockName,
@@ -723,6 +847,20 @@ struct Allocator {
}
private:
FunctionPointer getAllocate() { return getFunctionPointer(Field::Allocate); }
FunctionPointer getDeallocate() {
return getFunctionPointer(Field::Deallocate);
}
FunctionPointer getAllocateFrame() {
return getFunctionPointer(Field::AllocateFrame);
}
FunctionPointer getDeallocateFrame() {
return getFunctionPointer(Field::DeallocateFrame);
}
FunctionPointer getFunctionPointer(Field field) {
llvm::Value *callee = getField(field);
if (auto &schema = field.getSchema(IGF.IGM)) {
@@ -732,18 +870,22 @@ private:
}
return FunctionPointer::createUnsigned(
FunctionPointer::Kind::Function, callee,
Signature(field.getFunctionType(IGF.IGM), {}, IGF.IGM.SwiftCC));
Signature(field.getFunctionType(IGF.IGM),
field.getFunctionAttributeList(IGF.IGM), IGF.IGM.SwiftCC));
}
};
} // end anonymous namespace
llvm::Constant *swift::irgen::getCoroAllocFn(IRGenModule &IGM) {
auto isSwiftCoroCCAvailable = IGM.SwiftCoroCC == llvm::CallingConv::SwiftCoro;
llvm::Constant *getCoroAllocFn(AllocationKind kind, IRGenModule &IGM) {
return IGM.getOrCreateHelperFunction(
"_swift_coro_alloc", IGM.Int8PtrTy, {IGM.CoroAllocatorPtrTy, IGM.SizeTy},
[isSwiftCoroCCAvailable](IRGenFunction &IGF) {
kind.getAllocationFunctionName(), IGM.CoroAllocationTy,
{IGM.CoroAllocatorPtrTy, IGM.CoroAllocationTy, IGM.SizeTy},
[kind](IRGenFunction &IGF) {
auto isSwiftCoroCCAvailable =
IGF.IGM.SwiftCoroCC == llvm::CallingConv::SwiftCoro;
auto parameters = IGF.collectParameters();
auto allocator = Allocator(parameters.claimNext(), IGF);
auto *frame = parameters.claimNext();
auto *allocatorValue = parameters.claimNext();
auto allocator = Allocator(allocatorValue, IGF);
auto *size = parameters.claimNext();
if (isSwiftCoroCCAvailable) {
// swiftcorocc is available, so if there's no allocator pointer,
@@ -761,8 +903,8 @@ llvm::Constant *swift::irgen::getCoroAllocFn(IRGenModule &IGM) {
IGF.Builder.CreateRet(alloca);
});
}
auto fnPtr = allocator.getAllocate();
auto *call = IGF.Builder.CreateCall(fnPtr, {size});
auto fnPtr = allocator.getAllocate(kind);
auto *call = IGF.Builder.CreateCall(fnPtr, {frame, allocatorValue, size});
call->setDoesNotThrow();
call->setCallingConv(IGF.IGM.SwiftCC);
IGF.Builder.CreateRet(call);
@@ -773,18 +915,21 @@ llvm::Constant *swift::irgen::getCoroAllocFn(IRGenModule &IGM) {
/*optionalLinkageOverride=*/nullptr, IGM.SwiftCoroCC,
/*transformAttributes=*/
[&IGM](llvm::AttributeList &attrs) {
IGM.addSwiftCoroAttributes(attrs, 0);
IGM.addSwiftCoroAttributes(attrs, 1);
});
}
llvm::Constant *swift::irgen::getCoroDeallocFn(IRGenModule &IGM) {
auto isSwiftCoroCCAvailable = IGM.SwiftCoroCC == llvm::CallingConv::SwiftCoro;
llvm::Constant *getCoroDeallocFn(AllocationKind kind, IRGenModule &IGM) {
return IGM.getOrCreateHelperFunction(
"_swift_coro_dealloc", IGM.VoidTy,
{IGM.CoroAllocatorPtrTy, IGM.Int8PtrTy},
[isSwiftCoroCCAvailable](IRGenFunction &IGF) {
kind.getDeallocationFunctionName(), IGM.VoidTy,
{IGM.CoroAllocatorPtrTy, IGM.CoroAllocationTy, IGM.CoroAllocationTy},
[kind](IRGenFunction &IGF) {
auto isSwiftCoroCCAvailable =
IGF.IGM.SwiftCoroCC == llvm::CallingConv::SwiftCoro;
auto parameters = IGF.collectParameters();
auto allocator = Allocator(parameters.claimNext(), IGF);
auto *frame = parameters.claimNext();
auto *allocatorValue = parameters.claimNext();
auto allocator = Allocator(allocatorValue, IGF);
auto *ptr = parameters.claimNext();
if (isSwiftCoroCCAvailable) {
// swiftcorocc is available, so if there's no allocator pointer,
@@ -815,8 +960,8 @@ llvm::Constant *swift::irgen::getCoroDeallocFn(IRGenModule &IGM) {
IGF.Builder.CreateRetVoid();
// Start emitting the "normal" block.
IGF.Builder.emitBlock(normalBlock);
auto fnPtr = allocator.getDeallocate();
auto *call = IGF.Builder.CreateCall(fnPtr, {ptr});
auto fnPtr = allocator.getDeallocate(kind);
auto *call = IGF.Builder.CreateCall(fnPtr, {frame, allocatorValue, ptr});
call->setDoesNotThrow();
call->setCallingConv(IGF.IGM.SwiftCC);
IGF.Builder.CreateRetVoid();
@@ -827,78 +972,163 @@ llvm::Constant *swift::irgen::getCoroDeallocFn(IRGenModule &IGM) {
/*optionalLinkageOverride=*/nullptr, IGM.SwiftCoroCC,
/*transformAttributes=*/
[&IGM](llvm::AttributeList &attrs) {
IGM.addSwiftCoroAttributes(attrs, 0);
IGM.addSwiftCoroAttributes(attrs, 1);
});
}
} // end anonymous namespace
llvm::Constant *swift::irgen::getCoroAllocFn(IRGenModule &IGM) {
return getCoroAllocFn(AllocationKind::Normal, IGM);
}
llvm::Constant *swift::irgen::getCoroAllocFrameFn(IRGenModule &IGM) {
return getCoroAllocFn(AllocationKind::Frame, IGM);
}
llvm::Constant *swift::irgen::getCoroDeallocFn(IRGenModule &IGM) {
return getCoroDeallocFn(AllocationKind::Normal, IGM);
}
llvm::Constant *swift::irgen::getCoroDeallocFrameFn(IRGenModule &IGM) {
return getCoroDeallocFn(AllocationKind::Frame, IGM);
}
using GetAllocFunctionPointer = FunctionPointer (IRGenModule::*)();
static llvm::Constant *
getAddrOfSwiftCoroAllocThunk(StringRef name,
GetAllocFunctionPointer getAlloc,
IRGenModule &IGM) {
auto *ty = IGM.CoroAllocateFnTy;
return IGM.getOrCreateHelperFunction(
name, ty->getReturnType(), ty->params(),
[getAlloc](IRGenFunction &IGF) {
auto parameters = IGF.collectParameters();
parameters.claimNext(); // frame
parameters.claimNext(); // allocator
auto *size = parameters.claimNext();
auto alloc = (IGF.IGM.*getAlloc)();
auto *call = IGF.Builder.CreateCall(alloc, {size});
IGF.Builder.CreateRet(call);
},
/*setIsNoInline=*/true, /*forPrologue=*/false,
/*isPerformanceConstraint=*/false,
/*optionalLinkage=*/nullptr,
/*specialCallingConv=*/std::nullopt,
/*transformAttributes=*/
[&IGM](llvm::AttributeList &attrs) {
IGM.addSwiftCoroAttributes(attrs, 1);
});
}
using GetDeallocFunctionPointer = FunctionPointer (IRGenModule::*)();
static llvm::Constant *
getAddrOfSwiftCoroDeallocThunk(StringRef name,
GetDeallocFunctionPointer getDealloc,
IRGenModule &IGM) {
auto *ty = IGM.CoroDeallocateFnTy;
return IGM.getOrCreateHelperFunction(
name, ty->getReturnType(), ty->params(),
[getDealloc](IRGenFunction &IGF) {
auto parameters = IGF.collectParameters();
parameters.claimNext(); // frame
parameters.claimNext(); // allocator
auto *ptr = parameters.claimNext();
auto dealloc = (IGF.IGM.*getDealloc)();
IGF.Builder.CreateCall(dealloc, {ptr});
IGF.Builder.CreateRetVoid();
},
/*setIsNoInline=*/true, /*forPrologue=*/false,
/*isPerformanceConstraint=*/false,
/*optionalLinkage=*/nullptr,
/*specialCallingConv=*/std::nullopt,
/*transformAttributes=*/
[&IGM](llvm::AttributeList &attrs) {
IGM.addSwiftCoroAttributes(attrs, 1);
});
}
static llvm::Constant *getAddrOfGlobalCoroAllocator(
IRGenModule &IGM, CoroAllocatorKind kind, bool shouldDeallocateImmediately,
llvm::Constant *allocFn, llvm::Constant *deallocFn) {
llvm::Constant *allocFn, llvm::Constant *deallocFn,
llvm::Constant *allocFrameFn, llvm::Constant *deallocFrameFn) {
auto entity = LinkEntity::forCoroAllocator(kind);
auto taskAllocator = IGM.getOrCreateLazyGlobalVariable(
entity,
[&](ConstantInitBuilder &builder) -> ConstantInitFuture {
auto allocator = builder.beginStruct(IGM.CoroAllocatorTy);
auto flags = CoroAllocatorFlags(kind);
flags.setShouldDeallocateImmediately(shouldDeallocateImmediately);
allocator.addInt32(flags.getOpaqueValue());
allocator.addSignedPointer(
allocFn, IGM.getOptions().PointerAuth.CoroAllocationFunction,
PointerAuthEntity::Special::CoroAllocationFunction);
allocator.addSignedPointer(
deallocFn, IGM.getOptions().PointerAuth.CoroDeallocationFunction,
PointerAuthEntity::Special::CoroDeallocationFunction);
return allocator.finishAndCreateFuture();
return Allocator::buildStruct(
builder, IGM, [&](auto field) -> llvm::Constant * {
switch (field) {
case Allocator::Field::Flags: {
auto flags = CoroAllocatorFlags(kind);
flags.setShouldDeallocateImmediately(
shouldDeallocateImmediately);
return llvm::ConstantInt::get(IGM.Int32Ty,
flags.getOpaqueValue());
}
case Allocator::Field::Allocate:
return allocFn;
case Allocator::Field::Deallocate:
return deallocFn;
case Allocator::Field::AllocateFrame:
return allocFrameFn;
case Allocator::Field::DeallocateFrame:
return deallocFrameFn;
}
});
},
[&](llvm::GlobalVariable *var) { var->setConstant(true); });
return taskAllocator;
}
static llvm::Constant *getAddrOfSwiftCCMalloc(IRGenModule &IGM) {
auto mallocFnPtr = IGM.getMallocFunctionPointer();
auto sig = mallocFnPtr.getSignature();
if (sig.getCallingConv() == IGM.SwiftCC) {
return IGM.getMallocFn();
}
return IGM.getOrCreateHelperFunction(
"_swift_malloc", sig.getType()->getReturnType(), sig.getType()->params(),
[](IRGenFunction &IGF) {
auto parameters = IGF.collectParameters();
auto *size = parameters.claimNext();
auto malloc = IGF.IGM.getMallocFunctionPointer();
auto *call = IGF.Builder.CreateCall(malloc, {size});
IGF.Builder.CreateRet(call);
});
static llvm::Constant *getAddrOfSwiftCoroMalloc(
IRGenModule &IGM) {
return getAddrOfSwiftCoroAllocThunk("_swift_coro_malloc",
&IRGenModule::getMallocFunctionPointer,
IGM);
}
static llvm::Constant *getAddrOfSwiftCCFree(IRGenModule &IGM) {
auto freeFnPtr = IGM.getFreeFunctionPointer();
auto sig = freeFnPtr.getSignature();
if (sig.getCallingConv() == IGM.SwiftCC) {
return IGM.getFreeFn();
}
return IGM.getOrCreateHelperFunction(
"_swift_free", sig.getType()->getReturnType(), sig.getType()->params(),
[](IRGenFunction &IGF) {
auto parameters = IGF.collectParameters();
auto *ptr = parameters.claimNext();
auto free = IGF.IGM.getFreeFunctionPointer();
IGF.Builder.CreateCall(free, {ptr});
IGF.Builder.CreateRetVoid();
});
static llvm::Constant *getAddrOfSwiftCoroFree(
IRGenModule &IGM) {
return getAddrOfSwiftCoroDeallocThunk("_swift_coro_free",
&IRGenModule::getFreeFunctionPointer,
IGM);
}
llvm::Constant *IRGenModule::getAddrOfGlobalCoroMallocAllocator() {
return getAddrOfGlobalCoroAllocator(*this, CoroAllocatorKind::Malloc,
/*shouldDeallocateImmediately=*/true,
getAddrOfSwiftCCMalloc(*this),
getAddrOfSwiftCCFree(*this));
return getAddrOfGlobalCoroAllocator(
*this, CoroAllocatorKind::Malloc,
/*shouldDeallocateImmediately=*/true,
getAddrOfSwiftCoroMalloc(*this),
getAddrOfSwiftCoroFree(*this),
getAddrOfSwiftCoroMalloc(*this),
getAddrOfSwiftCoroFree(*this));
}
static llvm::Constant *
getAddrOfSwiftCoroTaskAlloc(IRGenModule &IGM) {
return getAddrOfSwiftCoroAllocThunk("_swift_coro_task_alloc",
&IRGenModule::getTaskAllocFunctionPointer,
IGM);
}
static llvm::Constant *
getAddrOfSwiftCoroTaskDealloc(IRGenModule &IGM) {
return getAddrOfSwiftCoroDeallocThunk(
"_swift_coro_task_dealloc",
&IRGenModule::getTaskDeallocFunctionPointer, IGM);
}
llvm::Constant *IRGenModule::getAddrOfGlobalCoroAsyncTaskAllocator() {
return getAddrOfGlobalCoroAllocator(*this, CoroAllocatorKind::Async,
/*shouldDeallocateImmediately=*/false,
getTaskAllocFn(), getTaskDeallocFn());
return getAddrOfGlobalCoroAllocator(
*this, CoroAllocatorKind::Async,
/*shouldDeallocateImmediately=*/false,
getAddrOfSwiftCoroTaskAlloc(*this),
getAddrOfSwiftCoroTaskDealloc(*this),
getAddrOfSwiftCoroTaskAlloc(*this),
getAddrOfSwiftCoroTaskDealloc(*this));
}
llvm::Value *

2
lib/IRGen/GenCoro.h
View File

@@ -16,7 +16,9 @@ namespace irgen {
class IRGenModule;
llvm::Constant *getCoroAllocFn(IRGenModule &IGM);
llvm::Constant *getCoroAllocFrameFn(IRGenModule &IGM);
llvm::Constant *getCoroDeallocFn(IRGenModule &IGM);
llvm::Constant *getCoroDeallocFrameFn(IRGenModule &IGM);
llvm::Value *
emitYieldOnce2CoroutineAllocator(IRGenFunction &IGF,

28
lib/IRGen/GenOpaque.cpp
View File

@@ -559,14 +559,15 @@ irgen::emitInitializeBufferWithCopyOfBufferCall(IRGenFunction &IGF,
StackAddress IRGenFunction::emitDynamicAlloca(SILType T,
const llvm::Twine &name) {
llvm::Value *size = emitLoadOfSize(*this, T);
return emitDynamicAlloca(IGM.Int8Ty, size, Alignment(16), true, name);
return emitDynamicAlloca(IGM.Int8Ty, size, Alignment(16), AllowsTaskAlloc,
IsNotForCalleeCoroutineFrame, name);
}
StackAddress IRGenFunction::emitDynamicAlloca(llvm::Type *eltTy,
llvm::Value *arraySize,
Alignment align,
bool allowTaskAlloc,
const llvm::Twine &name) {
StackAddress IRGenFunction::emitDynamicAlloca(
llvm::Type *eltTy, llvm::Value *arraySize, Alignment align,
AllowsTaskAlloc_t allowTaskAlloc,
IsForCalleeCoroutineFrame_t forCalleeCoroutineFrame,
const llvm::Twine &name) {
// Async functions call task alloc.
if (allowTaskAlloc && isAsync()) {
llvm::Value *byteCount;
@@ -600,9 +601,10 @@ StackAddress IRGenFunction::emitDynamicAlloca(llvm::Type *eltTy,
auto alignment = llvm::ConstantInt::get(IGM.Int32Ty, align.getValue());
// Allocate memory. This produces an abstract token.
auto allocToken =
Builder.CreateIntrinsicCall(llvm::Intrinsic::coro_alloca_alloc,
{IGM.SizeTy}, {byteCount, alignment});
auto *allocToken = Builder.CreateIntrinsicCall(
forCalleeCoroutineFrame ? llvm::Intrinsic::coro_alloca_alloc_frame
: llvm::Intrinsic::coro_alloca_alloc,
{IGM.SizeTy}, {byteCount, alignment});
// Get the allocation result.
auto ptr = Builder.CreateIntrinsicCall(llvm::Intrinsic::coro_alloca_get,
@@ -642,7 +644,8 @@ StackAddress IRGenFunction::emitDynamicAlloca(llvm::Type *eltTy,
/// location before the dynamic alloca's call.
void IRGenFunction::emitDeallocateDynamicAlloca(StackAddress address,
bool allowTaskDealloc,
bool useTaskDeallocThrough) {
bool useTaskDeallocThrough,
bool forCalleeCoroutineFrame) {
// Async function use taskDealloc.
if (allowTaskDealloc && isAsync() && address.getAddress().isValid()) {
if (useTaskDeallocThrough) {
@@ -670,7 +673,10 @@ void IRGenFunction::emitDeallocateDynamicAlloca(StackAddress address,
#endif
return;
}
Builder.CreateIntrinsicCall(llvm::Intrinsic::coro_alloca_free, allocToken);
Builder.CreateIntrinsicCall(forCalleeCoroutineFrame
? llvm::Intrinsic::coro_alloca_free_frame
: llvm::Intrinsic::coro_alloca_free,
allocToken);
return;
}
// Otherwise, call llvm.stackrestore if an address was saved.

36
lib/IRGen/GenPack.cpp
View File

@@ -467,9 +467,9 @@ irgen::emitTypeMetadataPack(IRGenFunction &IGF, CanPackType packType,
}
assert(packType->containsPackExpansionType());
auto pack = IGF.emitDynamicAlloca(IGF.IGM.TypeMetadataPtrTy, shape,
IGF.IGM.getPointerAlignment(),
/*allowTaskAlloc=*/true);
auto pack =
IGF.emitDynamicAlloca(IGF.IGM.TypeMetadataPtrTy, shape,
IGF.IGM.getPointerAlignment(), AllowsTaskAlloc);
auto visitFn =
[&](CanType eltTy, unsigned staticIndex,
@@ -612,9 +612,9 @@ irgen::emitWitnessTablePack(IRGenFunction &IGF, CanPackType packType,
}
assert(packType->containsPackExpansionType());
auto pack = IGF.emitDynamicAlloca(IGF.IGM.WitnessTablePtrTy, shape,
IGF.IGM.getPointerAlignment(),
/*allowTaskAlloc=*/true);
auto pack =
IGF.emitDynamicAlloca(IGF.IGM.WitnessTablePtrTy, shape,
IGF.IGM.getPointerAlignment(), AllowsTaskAlloc);
auto index = 0;
auto visitFn = [&](CanType eltTy, unsigned staticIndex,
@@ -1184,9 +1184,9 @@ StackAddress irgen::allocatePack(IRGenFunction &IGF, CanSILPackType packType) {
}
assert(packType->containsPackExpansionType());
auto addr = IGF.emitDynamicAlloca(IGF.IGM.OpaquePtrTy, shape,
IGF.IGM.getPointerAlignment(),
/*allowTaskAlloc=*/true);
auto addr =
IGF.emitDynamicAlloca(IGF.IGM.OpaquePtrTy, shape,
IGF.IGM.getPointerAlignment(), AllowsTaskAlloc);
return addr;
}
@@ -1255,10 +1255,9 @@ irgen::emitDynamicTupleTypeLabels(IRGenFunction &IGF, CanTupleType type,
llvm::ConstantInt::get(IGF.IGM.SizeTy, 1));
// Allocate space for the label string; we fill it in below.
StackAddress labelString = IGF.emitDynamicAlloca(
IGF.IGM.Int8Ty, labelLength,
IGF.IGM.getPointerAlignment(),
/*allowTaskAlloc=*/true);
StackAddress labelString =
IGF.emitDynamicAlloca(IGF.IGM.Int8Ty, labelLength,
IGF.IGM.getPointerAlignment(), AllowsTaskAlloc);
// Get the static label string, where each pack expansion is one element.
auto *staticLabelString = getTupleLabelsString(IGF.IGM, type);
@@ -1371,9 +1370,8 @@ irgen::emitDynamicFunctionParameterFlags(IRGenFunction &IGF,
AnyFunctionType::CanParamArrayRef params,
CanPackType packType,
llvm::Value *shapeExpression) {
auto array =
IGF.emitDynamicAlloca(IGF.IGM.Int32Ty, shapeExpression,
Alignment(4), /*allowTaskAlloc=*/true);
auto array = IGF.emitDynamicAlloca(IGF.IGM.Int32Ty, shapeExpression,
Alignment(4), AllowsTaskAlloc);
unsigned numExpansions = 0;
@@ -1431,9 +1429,9 @@ irgen::emitInducedTupleTypeMetadataPack(
IRGenFunction &IGF, llvm::Value *tupleMetadata) {
auto *shape = emitTupleTypeMetadataLength(IGF, tupleMetadata);
auto pack = IGF.emitDynamicAlloca(IGF.IGM.TypeMetadataPtrTy, shape,
IGF.IGM.getPointerAlignment(),
/*allowTaskAlloc=*/true);
auto pack =
IGF.emitDynamicAlloca(IGF.IGM.TypeMetadataPtrTy, shape,
IGF.IGM.getPointerAlignment(), AllowsTaskAlloc);
auto elementForIndex =
[&](llvm::Value *index) -> llvm::Value * {
return irgen::emitTupleTypeMetadataElementType(IGF, tupleMetadata, index);

4
lib/IRGen/GenPointerAuth.cpp
View File

@@ -399,6 +399,10 @@ PointerAuthEntity::getDeclDiscriminator(IRGenModule &IGM) const {
return SpecialPointerAuthDiscriminators::CoroAllocationFunction;
case Special::CoroDeallocationFunction:
return SpecialPointerAuthDiscriminators::CoroDeallocationFunction;
case Special::CoroFrameAllocationFunction:
return SpecialPointerAuthDiscriminators::CoroFrameAllocationFunction;
case Special::CoroFrameDeallocationFunction:
return SpecialPointerAuthDiscriminators::CoroFrameDeallocationFunction;
}
llvm_unreachable("bad kind");
};

2
lib/IRGen/GenPointerAuth.h
View File

@@ -70,6 +70,8 @@ public:
TypeLayoutString,
CoroAllocationFunction,
CoroDeallocationFunction,
CoroFrameAllocationFunction,
CoroFrameDeallocationFunction,
};
private:

8
lib/IRGen/IRGen.cpp
View File

@@ -1098,6 +1098,14 @@ static void setPointerAuthOptions(PointerAuthOptions &opts,
opts.CoroDeallocationFunction = PointerAuthSchema(
codeKey, /*address*/ false, Discrimination::Constant,
SpecialPointerAuthDiscriminators::CoroDeallocationFunction);
opts.CoroAllocationFunction = PointerAuthSchema(
codeKey, /*address*/ false, Discrimination::Constant,
SpecialPointerAuthDiscriminators::CoroFrameAllocationFunction);
opts.CoroDeallocationFunction = PointerAuthSchema(
codeKey, /*address*/ false, Discrimination::Constant,
SpecialPointerAuthDiscriminators::CoroFrameDeallocationFunction);
}
std::unique_ptr<llvm::TargetMachine>

22
lib/IRGen/IRGenFunction.h
View File

@@ -66,6 +66,16 @@ namespace irgen {
class TypeInfo;
enum class ValueWitness : unsigned;
enum AllowsTaskAlloc_t : bool {
DoesNotAllowTaskAlloc = false,
AllowsTaskAlloc = true,
};
enum IsForCalleeCoroutineFrame_t : bool {
IsNotForCalleeCoroutineFrame,
IsForCalleeCoroutineFrame,
};
/// IRGenFunction - Primary class for emitting LLVM instructions for a
/// specific function.
class IRGenFunction {
@@ -302,12 +312,16 @@ public:
const llvm::Twine &name = "");
StackAddress emitDynamicAlloca(SILType type, const llvm::Twine &name = "");
StackAddress emitDynamicAlloca(llvm::Type *eltTy, llvm::Value *arraySize,
Alignment align, bool allowTaskAlloc = true,
const llvm::Twine &name = "");
StackAddress
emitDynamicAlloca(llvm::Type *eltTy, llvm::Value *arraySize, Alignment align,
AllowsTaskAlloc_t allowTaskAlloc = AllowsTaskAlloc,
IsForCalleeCoroutineFrame_t forCalleeCoroutineFrame =
IsNotForCalleeCoroutineFrame,
const llvm::Twine &name = "");
void emitDeallocateDynamicAlloca(StackAddress address,
bool allowTaskDealloc = true,
bool useTaskDeallocThrough = false);
bool useTaskDeallocThrough = false,
bool forCalleeCoroutineFrame = false);
llvm::BasicBlock *createBasicBlock(const llvm::Twine &Name);
const TypeInfo &getTypeInfoForUnlowered(Type subst);

14
lib/IRGen/IRGenModule.cpp
View File

@@ -772,17 +772,19 @@ IRGenModule::IRGenModule(IRGenerator &irgen,
CoroFunctionPointerTy = createStructType(*this, "swift.coro_func_pointer",
{RelativeAddressTy, Int32Ty}, true);
CoroAllocateFnTy =
llvm::FunctionType::get(CoroAllocationTy, SizeTy, /*isVarArg*/ false);
CoroDeallocateFnTy =
llvm::FunctionType::get(VoidTy, CoroAllocationTy, /*isVarArg*/ false);
CoroAllocateFnTy = llvm::FunctionType::get(
CoroAllocationTy, {CoroAllocationTy, CoroAllocatorPtrTy, SizeTy}, /*isVarArg*/ false);
CoroDeallocateFnTy = llvm::FunctionType::get(
VoidTy, {CoroAllocationTy, CoroAllocatorPtrTy, CoroAllocationTy}, /*isVarArg*/ false);
CoroAllocatorFlagsTy = Int32Ty;
// swift/ABI/Coro.h : CoroAllocator
CoroAllocatorTy = createStructType(*this, "swift.coro_allocator",
{
Int32Ty, // CoroAllocator.Flags
PtrTy,
PtrTy,
PtrTy, // allocate
PtrTy, // deallocate
PtrTy, // allocateFrame
PtrTy, // deallocateFrame
});
SwiftImplicitActorType =
createStructType(*this, "swift.implicit_isolated_actor_type",

5
lib/IRGen/IRGenSIL.cpp
View File

@@ -3726,8 +3726,9 @@ static void emitBuiltinStackAlloc(IRGenSILFunction &IGF,
return;
}
auto stackAddress = IGF.emitDynamicAlloca(IGF.IGM.Int8Ty, size, align,
false, "temp_alloc");
auto stackAddress =
IGF.emitDynamicAlloca(IGF.IGM.Int8Ty, size, align, DoesNotAllowTaskAlloc,
IsNotForCalleeCoroutineFrame, "temp_alloc");
IGF.setLoweredStackAddress(i, stackAddress);
}

34
test/IRGen/coroutine_accessors.swift
View File

@@ -23,15 +23,15 @@
// CHECK-SAME: i32 0
// CHECK-SAME: }>
// CHECK-arm64e-LABEL: _swift_malloc.ptrauth = private constant {
// CHECK-arm64e-SAME: ptr @_swift_malloc,
// CHECK-arm64e-LABEL: _swift_coro_malloc.ptrauth = private constant {
// CHECK-arm64e-SAME: ptr @_swift_coro_malloc,
// CHECK-arm64e-SAME: i32 0,
// CHECK-arm64e-SAME: i64 0,
// CHECK-arm64e-SAME: i64 24469 }
// CHECK-arm64e-SAME: section "llvm.ptrauth"
// CHECK-arm64e-SAME: align 8
// CHECK-arm64e-LABEL: _swift_free.ptrauth = private constant {
// CHECK-arm64e-SAME: ptr @_swift_free,
// CHECK-arm64e-LABEL: _swift_coro_free.ptrauth = private constant {
// CHECK-arm64e-SAME: ptr @_swift_coro_free,
// CHECK-arm64e-SAME: i32 0,
// CHECK-arm64e-SAME: i64 0,
// CHECK-arm64e-SAME: i64 40879 },
@@ -42,15 +42,15 @@
// CHECK-SAME: malloc
// CHECK-SAME: free
// CHECK-SAME: }
// CHECK-arm64e-LABEL: swift_task_alloc.ptrauth = private constant {
// CHECK-arm64e-SAME: ptr @swift_task_alloc,
// CHECK-arm64e-LABEL: _swift_coro_task_alloc.ptrauth = private constant {
// CHECK-arm64e-SAME: ptr @_swift_coro_task_alloc,
// CHECK-arm64e-SAME: i32 0,
// CHECK-arm64e-SAME: i64 0,
// CHECK-arm64e-SAME: i64 24469 }
// CHECK-arm64e-SAME: section "llvm.ptrauth"
// CHECK-arm64e-SAME: align 8
// CHECK-arm64e-LABEL: @swift_task_dealloc.ptrauth = private constant {
// CHECK-arm64e-SAME: ptr @swift_task_dealloc,
// CHECK-arm64e-LABEL: @_swift_coro_task_dealloc.ptrauth = private constant {
// CHECK-arm64e-SAME: ptr @_swift_coro_task_dealloc,
// CHECK-arm64e-SAME: i32 0,
// CHECK-arm64e-SAME: i64 0,
// CHECK-arm64e-SAME: i64 40879 },
@@ -58,11 +58,12 @@
// CHECK-arm64e-SAME: align 8
// CHECK-LABEL: _swift_coro_async_allocator = linkonce_odr hidden constant %swift.coro_allocator {
// CHECK-SAME: i32 1,
// CHECK-SAME: swift_task_alloc
// CHECK-SAME: swift_task_dealloc
// CHECK-SAME: _swift_coro_task_alloc
// CHECK-SAME: _swift_coro_task_dealloc
// CHECK-SAME: }
// CHECK-LABEL: @_swift_coro_alloc(
// CHECK-SAME: ptr [[FRAME:%[^,]+]]
// CHECK-SAME: ptr swiftcoro [[ALLOCATOR:%[^,]+]]
// CHECK-SAME: [[INT]] [[SIZE:%[^)]+]]
// CHECK-SAME: )
@@ -76,11 +77,12 @@
// CHECK-arm64e: [[ALLOCATE_FN_BITS:%[^,]+]] = ptrtoint ptr [[ALLOCATE_FN]] to i64
// CHECK-arm64e: [[ALLOCATE_FN_BITS_AUTHED:%[^,]+]] = call i64 @llvm.ptrauth.auth(i64 [[ALLOCATE_FN_BITS]], i32 0, i64 24469)
// CHECK-arm64e: [[ALLOCATE_FN:%[^,]+]] = inttoptr i64 [[ALLOCATE_FN_BITS_AUTHED]]
// CHECK: [[ALLOCATION:%[^,]+]] = call swiftcc ptr [[ALLOCATE_FN]]([[INT]] [[SIZE]])
// CHECK: [[ALLOCATION:%[^,]+]] = call swiftcc ptr [[ALLOCATE_FN]](ptr [[FRAME]], ptr swiftcoro [[ALLOCATOR]], [[INT]] [[SIZE]])
// CHECK: ret ptr [[ALLOCATION]]
// CHECK: }
// CHECK-LABEL: @_swift_coro_dealloc(
// CHECK-SAME: ptr [[FRAME:%[^,]+]]
// CHECK-SAME: ptr swiftcoro [[ALLOCATOR:%[^,]+]]
// CHECK-SAME: ptr [[ADDRESS:%[^)]+]]
// CHECK-SAME: )
@@ -107,7 +109,7 @@
// CHECK-arm64e: [[DEALLOCATE_FN_BITS:%[^,]+]] = ptrtoint ptr [[DEALLOCATE_FN]] to i64
// CHECK-arm64e: [[DEALLOCATE_FN_BITS_AUTHED:%[^,]+]] = call i64 @llvm.ptrauth.auth(i64 [[DEALLOCATE_FN_BITS]], i32 0, i64 40879)
// CHECK-arm64e: [[DEALLOCATE_FN:%[^,]+]] = inttoptr i64 [[DEALLOCATE_FN_BITS_AUTHED]]
// CHECK: call swiftcc void [[DEALLOCATE_FN]](ptr [[ADDRESS]])
// CHECK: call swiftcc void [[DEALLOCATE_FN]](ptr [[FRAME]], ptr swiftcoro [[ALLOCATOR]], ptr [[ADDRESS]])
// CHECK: ret void
// CHECK: }
@@ -308,7 +310,7 @@ public var force_yield_once_convention : () {
// CHECK-SAME: _swift_coro_malloc_allocator
// CHECK-SAME: )
// CHECK: call void @llvm.lifetime.end.p0(i64 -1, ptr [[FRAME]])
// CHECK: call void @llvm.coro.alloca.free(token [[ALLOCATION]])
// CHECK: call void @llvm.coro.alloca.free.frame(token [[ALLOCATION]])
// CHECK: }
@_silgen_name("increment_irm_yield_once")
_modify {
@@ -338,8 +340,8 @@ public var force_yield_once_2_convention : () {
// : i32 1
// : )
// CHECK-64: [[SIZE_64:%[^,]+]] = zext i32 {{%[^,]+}} to i64
// CHECK-64: [[ALLOCATION:%[^,]+]] = call token @llvm.coro.alloca.alloc.i64(i64 [[SIZE_64]], i32 16)
// CHECK-32: [[ALLOCATION:%[^,]+]] = call token @llvm.coro.alloca.alloc.i32(i32 {{%[^,]+}}, i32 16)
// CHECK-64: [[ALLOCATION:%[^,]+]] = call token @llvm.coro.alloca.alloc.frame.i64(i64 [[SIZE_64]], i32 16)
// CHECK-32: [[ALLOCATION:%[^,]+]] = call token @llvm.coro.alloca.alloc.frame.i32(i32 {{%[^,]+}}, i32 16)
// CHECK: [[FRAME:%[^,]+]] = call ptr @llvm.coro.alloca.get(token [[ALLOCATION]])
// CHECK: call void @llvm.lifetime.start.p0(i64 -1, ptr [[FRAME]])
// CHECK: [[RAMP:%[^,]+]] = call ptr @llvm.coro.prepare.retcon(ptr @"$s19coroutine_accessors1SV3irmSivx")
@@ -357,7 +359,7 @@ public var force_yield_once_2_convention : () {
// CHECK-SAME: [[ALLOCATOR]]
// CHECK-SAME: )
// CHECK: call void @llvm.lifetime.end.p0(i64 -1, ptr [[FRAME]])
// CHECK: call void @llvm.coro.alloca.free(token [[ALLOCATION]])
// CHECK: call void @llvm.coro.alloca.free.frame(token [[ALLOCATION]])
// CHECK: }
@_silgen_name("increment_irm_yield_once_2")
modify {

36
test/IRGen/coroutine_accessors_popless.swift
View File

@@ -25,15 +25,15 @@
// CHECK-SAME: i32 0
// CHECK-SAME: }>
// CHECK-arm64e-LABEL: swift_task_alloc.ptrauth = private constant {
// CHECK-arm64e-SAME: ptr @swift_task_alloc,
// CHECK-arm64e-LABEL: _swift_coro_task_alloc.ptrauth = private constant {
// CHECK-arm64e-SAME: ptr @_swift_coro_task_alloc,
// CHECK-arm64e-SAME: i32 0,
// CHECK-arm64e-SAME: i64 0,
// CHECK-arm64e-SAME: i64 24469 }
// CHECK-arm64e-SAME: section "llvm.ptrauth"
// CHECK-arm64e-SAME: align 8
// CHECK-arm64e-LABEL: @swift_task_dealloc.ptrauth = private constant {
// CHECK-arm64e-SAME: ptr @swift_task_dealloc,
// CHECK-arm64e-LABEL: @_swift_coro_task_dealloc.ptrauth = private constant {
// CHECK-arm64e-SAME: ptr @_swift_coro_task_dealloc,
// CHECK-arm64e-SAME: i32 0,
// CHECK-arm64e-SAME: i64 0,
// CHECK-arm64e-SAME: i64 40879 },
@@ -41,18 +41,18 @@
// CHECK-arm64e-SAME: align 8
// CHECK-LABEL: _swift_coro_async_allocator = linkonce_odr hidden constant %swift.coro_allocator {
// CHECK-SAME: i32 1,
// CHECK-SAME: swift_task_alloc
// CHECK-SAME: swift_task_dealloc
// CHECK-SAME: _swift_coro_task_alloc
// CHECK-SAME: _swift_coro_task_dealloc
// CHECK-SAME: }
// CHECK-arm64e-LABEL: _swift_malloc.ptrauth = private constant {
// CHECK-arm64e-SAME: ptr @_swift_malloc,
// CHECK-arm64e-LABEL: _swift_coro_malloc.ptrauth = private constant {
// CHECK-arm64e-SAME: ptr @_swift_coro_malloc,
// CHECK-arm64e-SAME: i32 0,
// CHECK-arm64e-SAME: i64 0,
// CHECK-arm64e-SAME: i64 24469 }
// CHECK-arm64e-SAME: section "llvm.ptrauth"
// CHECK-arm64e-SAME: align 8
// CHECK-arm64e-LABEL: _swift_free.ptrauth = private constant {
// CHECK-arm64e-SAME: ptr @_swift_free,
// CHECK-arm64e-LABEL: _swift_coro_free.ptrauth = private constant {
// CHECK-arm64e-SAME: ptr @_swift_coro_free,
// CHECK-arm64e-SAME: i32 0,
// CHECK-arm64e-SAME: i64 0,
// CHECK-arm64e-SAME: i64 40879 },
@@ -65,7 +65,8 @@
// CHECK-SAME: }
// CHECK-LABEL: @_swift_coro_alloc(
// CHECK-SAME: ptr swiftcoro [[ALLOCATOR:%[^,]+]]
// CHECK-SAME: ptr [[FRAME:%[^,]+]]
// CHECK-SAME: ptr swiftcoro [[ALLOCATOR:%[^,]+]],
// CHECK-SAME: i64 [[SIZE:%[^)]+]]
// CHECK-SAME: )
// CHECK-SAME: {
@@ -87,12 +88,13 @@
// CHECK-arm64e: [[ALLOCATE_FN_BITS:%[^,]+]] = ptrtoint ptr [[ALLOCATE_FN]] to i64
// CHECK-arm64e: [[ALLOCATE_FN_BITS_AUTHED:%[^,]+]] = call i64 @llvm.ptrauth.auth(i64 [[ALLOCATE_FN_BITS]], i32 0, i64 24469)
// CHECK-arm64e: [[ALLOCATE_FN:%[^,]+]] = inttoptr i64 [[ALLOCATE_FN_BITS_AUTHED]]
// CHECK: [[ALLOCATION:%[^,]+]] = call swiftcc ptr [[ALLOCATE_FN]]([[INT]] [[SIZE]])
// CHECK: [[ALLOCATION:%[^,]+]] = call swiftcc ptr [[ALLOCATE_FN]](ptr [[FRAME]], ptr swiftcoro [[ALLOCATOR]], [[INT]] [[SIZE]])
// CHECK: ret ptr [[ALLOCATION]]
// CHECK: }
// CHECK-LABEL: @_swift_coro_dealloc(
// CHECK-SAME: ptr swiftcoro [[ALLOCATOR:%[^,]+]]
// CHECK-SAME: ptr [[FRAME:%[^,]+]]
// CHECK-SAME: ptr swiftcoro [[ALLOCATOR:%[^,]+]],
// CHECK-SAME: ptr [[ADDRESS:%[^)]+]]
// CHECK-SAME: )
// CHECK-SAME: {
@@ -125,7 +127,7 @@
// CHECK-arm64e: [[DEALLOCATE_FN_BITS:%[^,]+]] = ptrtoint ptr [[DEALLOCATE_FN]] to i64
// CHECK-arm64e: [[DEALLOCATE_FN_BITS_AUTHED:%[^,]+]] = call i64 @llvm.ptrauth.auth(i64 [[DEALLOCATE_FN_BITS]], i32 0, i64 40879)
// CHECK-arm64e: [[DEALLOCATE_FN:%[^,]+]] = inttoptr i64 [[DEALLOCATE_FN_BITS_AUTHED]]
// CHECK: call swiftcc void [[DEALLOCATE_FN]](ptr [[ADDRESS]])
// CHECK: call swiftcc void [[DEALLOCATE_FN]](ptr [[FRAME]], ptr swiftcoro [[ALLOCATOR]], ptr [[ADDRESS]])
// CHECK: ret void
// CHECK: }
@@ -255,7 +257,7 @@ public var force_yield_once_convention : () {
// CHECK-SAME: _swift_coro_malloc_allocator
// CHECK-SAME: )
// CHECK: call void @llvm.lifetime.end.p0(i64 -1, ptr [[FRAME]])
// CHECK: call void @llvm.coro.alloca.free(token [[ALLOCATION]])
// CHECK: call void @llvm.coro.alloca.free.frame(token [[ALLOCATION]])
// CHECK: }
@_silgen_name("increment_i_yield_once")
_modify {
@@ -284,7 +286,7 @@ public var force_yield_once_2_convention : () {
// : i32 1
// : )
// CHECK: [[SIZE:%[^,]+]] = zext i32 {{%[^,]+}} to i64
// CHECK: [[ALLOCATION:%[^,]+]] = call token @llvm.coro.alloca.alloc.i64(i64 [[SIZE]], i32 16)
// CHECK: [[ALLOCATION:%[^,]+]] = call token @llvm.coro.alloca.alloc.frame.i64(i64 [[SIZE]], i32 16)
// CHECK: [[FRAME:%[^,]+]] = call ptr @llvm.coro.alloca.get(token [[ALLOCATION]])
// CHECK: call void @llvm.lifetime.start.p0(i64 -1, ptr [[FRAME]])
// CHECK: [[RAMP:%[^,]+]] = call ptr @llvm.coro.prepare.retcon(ptr @"$s27coroutine_accessors_popless1iSivx")
@@ -302,7 +304,7 @@ public var force_yield_once_2_convention : () {
// CHECK-SAME: [[ALLOCATOR]]
// CHECK-SAME: )
// CHECK: call void @llvm.lifetime.end.p0(i64 -1, ptr [[FRAME]])
// CHECK: call void @llvm.coro.alloca.free(token [[ALLOCATION]])
// CHECK: call void @llvm.coro.alloca.free.frame(token [[ALLOCATION]])
// CHECK: }
@_silgen_name("increment_i_yield_once_2")
modify {