averello/swift-mirror
1
0
Fork 0
mirror of https://github.com/apple/swift.git synced 2025-12-14 20:36:38 +01:00
Code Issues Packages Projects Releases Wiki Activity
Files
coro-ast-no-diff
swift-mirror/lib/IRGen/GenOpaque.cpp
Nate Chandler e0ead75622 [CoroutineAccessors] Use typed-malloc.
2025-10-30 03:59:18 -07:00

1531 lines
62 KiB
C++
Raw Permalink Blame History

//===--- GenOpaque.cpp - Swift IR-generation for opaque values ------------===//
//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See https://swift.org/LICENSE.txt for license information
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
//
//===----------------------------------------------------------------------===//
//
// This file implements IR generation for opaque values and value
// witness operations.
//
// In the comments throughout this file, three type names are used:
// 'B' is the type of a fixed-size buffer
// 'T' is the type which implements a protocol
// 'W' is the type of a witness to the protocol
//
//===----------------------------------------------------------------------===//
#include "llvm/ADT/SmallString.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/IR/DerivedTypes.h"
#include "swift/AST/IRGenOptions.h"
#include "swift/ABI/MetadataValues.h"
#include "swift/Basic/Assertions.h"
#include "swift/IRGen/ValueWitness.h"
#include "swift/SIL/TypeLowering.h"
#include "Callee.h"
#include "Explosion.h"
#include "FixedTypeInfo.h"
#include "GenPointerAuth.h"
#include "IRGenDebugInfo.h"
#include "IRGenFunction.h"
#include "IRGenModule.h"
#include "ProtocolInfo.h"
#include "GenOpaque.h"
using namespace swift;
using namespace irgen;
/// A fixed-size buffer is always three pointers in size and pointer-aligned.
/// If we align them more, we'll need to introduce padding to
/// make protocol types work.
Size irgen::getFixedBufferSize(IRGenModule &IGM) {
return NumWords_ValueBuffer * IGM.getPointerSize();
}
Alignment irgen::getFixedBufferAlignment(IRGenModule &IGM) {
return IGM.getPointerAlignment();
}
/// Lazily create the standard fixed-buffer type.
llvm::Type *IRGenModule::getFixedBufferTy() {
if (FixedBufferTy) return FixedBufferTy;
auto size = getFixedBufferSize(*this).getValue();
FixedBufferTy = llvm::ArrayType::get(Int8Ty, size);
return FixedBufferTy;
}
static llvm::Type *createWitnessType(IRGenModule &IGM, ValueWitness index) {
switch (index) {
// void (*destroy)(T *object, witness_t *self);
case ValueWitness::Destroy: {
llvm::Type *args[] = { IGM.OpaquePtrTy, IGM.TypeMetadataPtrTy };
return llvm::FunctionType::get(IGM.VoidTy, args, /*isVarArg*/ false);
}
// T *(*initializeBufferWithCopyOfBuffer)(B *dest, B *src, M *self);
case ValueWitness::InitializeBufferWithCopyOfBuffer: {
auto *bufPtrTy = IGM.PtrTy;
llvm::Type *args[] = { bufPtrTy, bufPtrTy, IGM.TypeMetadataPtrTy };
return llvm::FunctionType::get(IGM.OpaquePtrTy, args, /*isVarArg*/ false);
}
// T *(*assignWithCopy)(T *dest, T *src, M *self);
// T *(*assignWithTake)(T *dest, T *src, M *self);
// T *(*initializeWithCopy)(T *dest, T *src, M *self);
// T *(*initializeWithTake)(T *dest, T *src, M *self);
case ValueWitness::AssignWithCopy:
case ValueWitness::AssignWithTake:
case ValueWitness::InitializeWithCopy:
case ValueWitness::InitializeWithTake: {
llvm::Type *ptrTy = IGM.OpaquePtrTy;
llvm::Type *args[] = { ptrTy, ptrTy, IGM.TypeMetadataPtrTy };
return llvm::FunctionType::get(ptrTy, args, /*isVarArg*/ false);
}
/// unsigned (*getEnumTag)(T *obj, M *self);
case ValueWitness::GetEnumTag: {
llvm::Type *ptrTy = IGM.OpaquePtrTy;
llvm::Type *metaTy = IGM.TypeMetadataPtrTy;
llvm::Type *indexTy = IGM.Int32Ty;
llvm::Type *args[] = {ptrTy, metaTy};
return llvm::FunctionType::get(indexTy, args, /*isVarArg*/ false);
}
/// void (*destructiveProjectEnumData)(T *obj, M *self);
case ValueWitness::DestructiveProjectEnumData: {
llvm::Type *voidTy = IGM.VoidTy;
llvm::Type *ptrTy = IGM.OpaquePtrTy;
llvm::Type *metaTy = IGM.TypeMetadataPtrTy;
llvm::Type *args[] = {ptrTy, metaTy};
return llvm::FunctionType::get(voidTy, args, /*isVarArg*/ false);
}
/// void (*destructiveInjectEnumTag)(T *obj, unsigned tag, M *self);
case ValueWitness::DestructiveInjectEnumTag: {
llvm::Type *voidTy = IGM.VoidTy;
llvm::Type *ptrTy = IGM.OpaquePtrTy;
llvm::Type *indexTy = IGM.Int32Ty;
llvm::Type *metaTy = IGM.TypeMetadataPtrTy;
llvm::Type *args[] = {ptrTy, indexTy, metaTy};
return llvm::FunctionType::get(voidTy, args, /*isVarArg*/ false);
}
/// unsigned (*getEnumTagSinglePayload)(const T* enum, UINT_TYPE emptyCases,
/// M *self)
case ValueWitness::GetEnumTagSinglePayload: {
llvm::Type *ptrTy = IGM.OpaquePtrTy;
llvm::Type *indexTy = IGM.Int32Ty;
llvm::Type *metaTy = IGM.TypeMetadataPtrTy;
llvm::Type *args[] = { ptrTy, indexTy, metaTy };
return llvm::FunctionType::get(indexTy, args, false);
}
/// void (*storeEnumTagSinglePayload)(T* enum, UINT_TYPE whichCase,
/// UINT_TYPE emptyCases,
/// M *self)
case ValueWitness::StoreEnumTagSinglePayload: {
llvm::Type *voidTy = IGM.VoidTy;
llvm::Type *ptrTy = IGM.OpaquePtrTy;
llvm::Type *indexTy = IGM.Int32Ty;
llvm::Type *metaTy = IGM.TypeMetadataPtrTy;
llvm::Type *args[] = { ptrTy, indexTy, indexTy, metaTy };
return llvm::FunctionType::get(voidTy, args, false);
}
case ValueWitness::Size:
case ValueWitness::Stride:
return IGM.SizeTy;
case ValueWitness::Flags:
case ValueWitness::ExtraInhabitantCount:
return IGM.Int32Ty;
}
llvm_unreachable("bad value witness!");
}
static llvm::AttributeList getValueWitnessAttrs(IRGenModule &IGM,
ValueWitness index) {
assert(isValueWitnessFunction(index));
auto &ctx = IGM.getLLVMContext();
// All value witnesses are nounwind.
auto attrs =
llvm::AttributeList().addFnAttribute(ctx, llvm::Attribute::NoUnwind);
switch (index) {
// These have two arguments, but they can alias.
case ValueWitness::AssignWithCopy:
return attrs;
// These have one argument.
case ValueWitness::Destroy:
case ValueWitness::DestructiveInjectEnumTag:
case ValueWitness::DestructiveProjectEnumData:
case ValueWitness::GetEnumTag:
case ValueWitness::StoreEnumTagSinglePayload:
return attrs.addParamAttribute(ctx, 0, llvm::Attribute::NoAlias);
case ValueWitness::GetEnumTagSinglePayload:
return attrs
.addFnAttribute(ctx, llvm::Attribute::getWithMemoryEffects(
ctx, llvm::MemoryEffects::readOnly()))
.addParamAttribute(ctx, 0, llvm::Attribute::NoAlias);
// These have two arguments and they don't alias each other.
case ValueWitness::AssignWithTake:
case ValueWitness::InitializeBufferWithCopyOfBuffer:
case ValueWitness::InitializeWithCopy:
case ValueWitness::InitializeWithTake:
return attrs.addParamAttribute(ctx, 0, llvm::Attribute::NoAlias)
.addParamAttribute(ctx, 1, llvm::Attribute::NoAlias);
case ValueWitness::Size:
case ValueWitness::Flags:
case ValueWitness::ExtraInhabitantCount:
case ValueWitness::Stride:
llvm_unreachable("not a function value witness");
}
llvm_unreachable("bad witness");
}
/// Return the cached pointer-to-function type for the given value
/// witness index.
llvm::Type *IRGenModule::getValueWitnessTy(ValueWitness index) {
assert(unsigned(index) < MaxNumValueWitnesses);
auto &ty = ValueWitnessTys[unsigned(index)];
if (ty) return ty;
ty = createWitnessType(*this, index);
return ty;
}
Signature IRGenModule::getValueWitnessSignature(ValueWitness index) {
assert(isValueWitnessFunction(index));
auto fnTy = cast<llvm::FunctionType>(getValueWitnessTy(index));
auto attrs = getValueWitnessAttrs(*this, index);
return Signature(fnTy, attrs, DefaultCC);
}
static StringRef getValueWitnessLabel(ValueWitness index) {
switch (index) {
case ValueWitness::Destroy:
return "destroy";
case ValueWitness::InitializeBufferWithCopyOfBuffer:
return "initializeBufferWithCopyOfBuffer";
case ValueWitness::AssignWithCopy:
return "assignWithCopy";
case ValueWitness::AssignWithTake:
return "assignWithTake";
case ValueWitness::InitializeWithCopy:
return "initializeWithCopy";
case ValueWitness::InitializeWithTake:
return "initializeWithTake";
case ValueWitness::Size:
return "size";
case ValueWitness::Flags:
return "flags";
case ValueWitness::ExtraInhabitantCount:
return "extraInhabitantCount";
case ValueWitness::Stride:
return "stride";
case ValueWitness::GetEnumTag:
return "getEnumTag";
case ValueWitness::DestructiveProjectEnumData:
return "destructiveProjectEnumData";
case ValueWitness::DestructiveInjectEnumTag:
return "destructiveInjectEnumTag";
case ValueWitness::GetEnumTagSinglePayload:
return "getEnumTagSinglePayload";
case ValueWitness::StoreEnumTagSinglePayload:
return "storeEnumTagSinglePayload";
}
llvm_unreachable("bad value witness index");
}
static llvm::StructType *
getOrCreateValueWitnessTableTy(IRGenModule &IGM, llvm::StructType *&cache,
StringRef name, bool includeEnumWitnesses) {
if (cache) return cache;
SmallVector<llvm::Type*, 16> types;
#define FUNC(lowerId, upperId, retTy, paramTys) \
types.push_back(IGM.Int8PtrTy);
#define DATA(lowerId, upperId, ty) \
types.push_back(IGM.getValueWitnessTy(ValueWitness::upperId));
// Add the base value witnesses.
#define WANT_ONLY_REQUIRED_VALUE_WITNESSES
#define FUNCTION_VALUE_WITNESS FUNC
#define DATA_VALUE_WITNESS DATA
#include "swift/ABI/ValueWitness.def"
// Add the enum value witnesses.
if (includeEnumWitnesses) {
#define WANT_ONLY_ENUM_VALUE_WITNESSES
#define FUNCTION_VALUE_WITNESS FUNC
#define DATA_VALUE_WITNESS DATA
#include "swift/ABI/ValueWitness.def"
}
#undef DATA
#undef FUNC
auto structTy = llvm::StructType::create(types, name);
cache = structTy;
return structTy;
}
llvm::PointerType *
IRGenModule::getOpaquePointerType(unsigned AddressSpace) const {
return llvm::PointerType::get(getLLVMContext(), AddressSpace);
}
llvm::StructType *IRGenModule::getValueWitnessTableTy() {
return getOrCreateValueWitnessTableTy(*this, ValueWitnessTableTy,
"swift.vwtable", false);
}
llvm::StructType *IRGenModule::getEnumValueWitnessTableTy() {
return getOrCreateValueWitnessTableTy(*this, EnumValueWitnessTableTy,
"swift.enum_vwtable", true);
}
Address irgen::slotForLoadOfOpaqueWitness(IRGenFunction &IGF,
llvm::Value *table,
WitnessIndex index,
bool areEntriesRelative) {
assert(table->getType() == IGF.IGM.WitnessTablePtrTy);
// Are we loading from a relative protocol witness table.
if (areEntriesRelative) {
llvm::Value *slot =
IGF.Builder.CreateBitOrPointerCast(table, IGF.IGM.RelativeAddressPtrTy);
if (index.getValue() != 0)
slot = IGF.Builder.CreateConstInBoundsGEP1_32(IGF.IGM.RelativeAddressTy,
slot, index.getValue());
return Address(slot, IGF.IGM.RelativeAddressTy, Alignment(4));
}
// GEP to the appropriate index, avoiding spurious IR in the trivial case.
llvm::Value *slot = table;
if (index.getValue() != 0)
slot = IGF.Builder.CreateConstInBoundsGEP1_32(IGF.IGM.WitnessTableTy, table,
index.getValue());
return Address(slot, IGF.IGM.WitnessTableTy, IGF.IGM.getPointerAlignment());
}
/// Load a specific witness from a known table. The result is
/// always an i8*.
llvm::Value *irgen::emitInvariantLoadOfOpaqueWitness(IRGenFunction &IGF,
bool isProtocolWitness,
llvm::Value *table,
WitnessIndex index,
llvm::Value **slotPtr) {
// Is this is a load of a relative protocol witness table entry.
auto isRelativeTable = IGF.IGM.IRGen.Opts.UseRelativeProtocolWitnessTables &&
isProtocolWitness;
auto slot = slotForLoadOfOpaqueWitness(IGF, table, index, isRelativeTable);
if (slotPtr) *slotPtr = slot.getAddress();
if (isRelativeTable) {
return IGF.emitLoadOfRelativePointer(slot, false, IGF.IGM.Int8Ty);
}
return IGF.emitInvariantLoad(slot);
}
/// Load a specific witness from a known table. The result is
/// always an i8*.
llvm::Value *irgen::emitInvariantLoadOfOpaqueWitness(IRGenFunction &IGF,
bool isProtocolWitness,
llvm::Value *table,
llvm::Value *index,
llvm::Value **slotPtr) {
assert(table->getType() == IGF.IGM.WitnessTablePtrTy);
assert(!isProtocolWitness &&
"This function does not yet support relative protocol witnesses");
// GEP to the appropriate index.
llvm::Value *slot =
IGF.Builder.CreateInBoundsGEP(IGF.IGM.WitnessTableTy, table, index);
if (slotPtr) *slotPtr = slot;
auto witness = IGF.Builder.CreateLoad(
Address(slot, IGF.IGM.WitnessTableTy, IGF.IGM.getPointerAlignment()));
IGF.setInvariantLoad(witness);
return witness;
}
static Address emitAddressOfValueWitnessTableValue(IRGenFunction &IGF,
llvm::Value *table,
ValueWitness witness) {
assert(!isValueWitnessFunction(witness));
assert(unsigned(witness) <= unsigned(ValueWitness::ExtraInhabitantCount) &&
"extraInhabitantCount not the last non-function value witness");
auto pointerSize = IGF.IGM.getPointerSize();
// Most of the witnesses are at an offset that's just a multiple of the
// pointer size, but the extra-inhabitant count is packed in after the
// 32-bit flags.
// This computation is correct for all pointer sizes, including 16.
// It would be wrong if size_t is ever a different size from the pointer
// size, though.
Size offset =
(witness == ValueWitness::ExtraInhabitantCount
? unsigned(ValueWitness::Flags) * pointerSize + Size(4)
: unsigned(witness) * pointerSize);
Address addr =
Address(table, IGF.IGM.WitnessTableTy, IGF.IGM.getPointerAlignment());
addr =
IGF.Builder.CreateElementBitCast(addr, IGF.IGM.getValueWitnessTableTy());
addr = IGF.Builder.CreateStructGEP(addr, unsigned(witness), offset);
return addr;
}
/// Given a value witness table, load one of the value witnesses.
/// The result has the appropriate type for the witness.
static llvm::Value *emitLoadOfValueWitnessValue(IRGenFunction &IGF,
llvm::Value *table,
ValueWitness witness) {
auto addr = emitAddressOfValueWitnessTableValue(IGF, table, witness);
auto load = IGF.Builder.CreateLoad(addr, getValueWitnessLabel(witness));
IGF.setInvariantLoad(load);
return load;
}
/// Given a type metadata pointer, load one of the value witnesses from its
/// value witness table.
static llvm::Value *
emitLoadOfValueWitnessValueFromMetadata(IRGenFunction &IGF,
llvm::Value *metadata,
ValueWitness index) {
llvm::Value *vwtable = IGF.emitValueWitnessTableRefForMetadata(metadata);
return emitLoadOfValueWitnessValue(IGF, vwtable, index);
}
/// Given a value witness table, load one of the value witnesses.
/// The result has the appropriate type for the witness.
static FunctionPointer emitLoadOfValueWitnessFunction(IRGenFunction &IGF,
llvm::Value *table,
ValueWitness index) {
assert(isValueWitnessFunction(index));
WitnessIndex windex = [&] {
unsigned i = unsigned(index);
if (i > unsigned(ValueWitness::Flags)) {
if (IGF.IGM.getPointerSize() == Size(8)) {
--i; // one pointer width skips both flags and xiCount
} else if (IGF.IGM.getPointerSize() == Size(4)) {
// no adjustment required
} else {
assert(IGF.IGM.getPointerSize() == Size(2));
i += 2; // flags and xiCount take up two pointers apiece
}
}
return WitnessIndex(i, false);
}();
llvm::Value *slot;
llvm::Value *witness =
emitInvariantLoadOfOpaqueWitness(IGF, /*isProtocolWitness*/false, table,
windex, &slot);
auto label = getValueWitnessLabel(index);
auto signature = IGF.IGM.getValueWitnessSignature(index);
witness = IGF.Builder.CreateBitCast(witness, IGF.IGM.PtrTy, label);
auto authInfo = PointerAuthInfo::emit(IGF,
IGF.getOptions().PointerAuth.ValueWitnesses,
slot, index);
witness->setName(getValueWitnessName(index));
return FunctionPointer::createSigned(FunctionPointer::Kind::Function, witness,
authInfo, signature);
}
/// Given a type metadata pointer, load one of the function
/// value witnesses from its value witness table.
static FunctionPointer
emitLoadOfValueWitnessFunctionFromMetadata(IRGenFunction &IGF,
llvm::Value *metadata,
ValueWitness index) {
llvm::Value *vwtable = IGF.emitValueWitnessTableRefForMetadata(metadata);
return emitLoadOfValueWitnessFunction(IGF, vwtable, index);
}
llvm::Value *IRGenFunction::emitValueWitnessValue(SILType type,
ValueWitness index) {
assert(!isValueWitnessFunction(index));
auto key = LocalTypeDataKind::forValueWitness(index);
if (auto witness = tryGetLocalTypeDataForLayout(type, key)) {
return witness;
}
auto vwtable = emitValueWitnessTableRef(type);
auto witness = emitLoadOfValueWitnessValue(*this, vwtable, index);
setScopedLocalTypeDataForLayout(type, key, witness);
return witness;
}
FunctionPointer
IRGenFunction::emitValueWitnessFunctionRef(SILType type,
llvm::Value *&metadataSlot,
ValueWitness index) {
assert(isValueWitnessFunction(index));
auto key = LocalTypeDataKind::forValueWitness(index);
if (auto witness = tryGetLocalTypeDataForLayout(type, key)) {
metadataSlot = emitTypeMetadataRefForLayout(type);
auto signature = IGM.getValueWitnessSignature(index);
PointerAuthInfo authInfo;
if (auto &schema = getOptions().PointerAuth.ValueWitnesses) {
auto discriminator =
tryGetLocalTypeDataForLayout(type,
LocalTypeDataKind::forValueWitnessDiscriminator(index));
assert(discriminator && "no saved discriminator for value witness fn!");
authInfo = PointerAuthInfo(schema.getKey(), discriminator);
}
return FunctionPointer::createSigned(FunctionPointer::Kind::Function,
witness, authInfo, signature);
}
auto vwtable = emitValueWitnessTableRef(type, &metadataSlot);
auto witness = emitLoadOfValueWitnessFunction(*this, vwtable, index);
setScopedLocalTypeDataForLayout(type, key, witness.getRawPointer());
if (auto &authInfo = witness.getAuthInfo()) {
setScopedLocalTypeDataForLayout(type,
LocalTypeDataKind::forValueWitnessDiscriminator(index),
authInfo.getDiscriminator());
}
return witness;
}
static llvm::Value *emitCastToOpaquePtr(IRGenFunction &IGF,
Address object) {
return IGF.Builder.CreateBitCast(object.getAddress(), IGF.IGM.OpaquePtrTy);
}
llvm::Value *irgen::emitInitializeBufferWithCopyOfBufferCall(IRGenFunction &IGF,
SILType T,
Address destBuffer,
Address srcBuffer) {
auto metadata = IGF.emitTypeMetadataRefForLayout(T);
return emitInitializeBufferWithCopyOfBufferCall(IGF, metadata,
destBuffer, srcBuffer);
}
/// Emit a call to do an 'initializeBufferWithCopyOfBuffer' operation.
llvm::Value *
irgen::emitInitializeBufferWithCopyOfBufferCall(IRGenFunction &IGF,
llvm::Value *metadata,
Address destBuffer,
Address srcBuffer) {
auto copyFn = emitLoadOfValueWitnessFunctionFromMetadata(IGF, metadata,
ValueWitness::InitializeBufferWithCopyOfBuffer);
llvm::CallInst *call =
IGF.Builder.CreateCall(copyFn,
{destBuffer.getAddress(), srcBuffer.getAddress(), metadata});
return call;
}
/// Emit a dynamic alloca call to allocate enough memory to hold an object of
/// type 'T' and an optional llvm.stackrestore point if 'isInEntryBlock' is
/// false.
StackAddress IRGenFunction::emitDynamicAlloca(SILType T,
const llvm::Twine &name) {
llvm::Value *size = emitLoadOfSize(*this, T);
return emitDynamicAlloca(IGM.Int8Ty, size, Alignment(16), AllowsTaskAlloc,
/*mallocTypeId=*/nullptr, name);
}
StackAddress IRGenFunction::emitDynamicAlloca(llvm::Type *eltTy,
llvm::Value *arraySize,
Alignment align,
AllowsTaskAlloc_t allowTaskAlloc,
llvm::Value *mallocTypeId,
const llvm::Twine &name) {
// Async functions call task alloc.
if (allowTaskAlloc && isAsync()) {
llvm::Value *byteCount;
auto eltSize = IGM.DataLayout.getTypeAllocSize(eltTy);
if (eltSize == 1) {
byteCount = arraySize;
} else {
byteCount = Builder.CreateMul(arraySize, IGM.getSize(Size(eltSize)));
}
// The task allocator wants size increments in the multiple of
// MaximumAlignment.
byteCount = alignUpToMaximumAlignment(IGM.SizeTy, byteCount);
auto address = emitTaskAlloc(byteCount, align);
auto stackAddress = StackAddress{address, address.getAddress()};
stackAddress = stackAddress.withAddress(
Builder.CreateElementBitCast(stackAddress.getAddress(), eltTy));
return stackAddress;
// In coroutines, call llvm.coro.alloca.alloc.
} else if (isCoroutine()) {
// NOTE: llvm does not support dynamic allocas in coroutines.
// Compute the number of bytes to allocate.
llvm::Value *byteCount;
auto eltSize = IGM.DataLayout.getTypeAllocSize(eltTy);
if (eltSize == 1) {
byteCount = arraySize;
} else {
byteCount = Builder.CreateMul(arraySize, IGM.getSize(Size(eltSize)));
}
auto alignment = llvm::ConstantInt::get(IGM.Int32Ty, align.getValue());
// Allocate memory. This produces an abstract token.
llvm::SmallVector<llvm::Value *, 4> args = {byteCount, alignment};
if (mallocTypeId) {
args.push_back(mallocTypeId);
}
auto *allocToken = Builder.CreateIntrinsicCall(
mallocTypeId ? llvm::Intrinsic::coro_alloca_alloc_frame
: llvm::Intrinsic::coro_alloca_alloc,
{IGM.SizeTy}, args);
// Get the allocation result.
auto ptr = Builder.CreateIntrinsicCall(llvm::Intrinsic::coro_alloca_get,
{allocToken});
auto stackAddress =
StackAddress{Address(ptr, IGM.Int8Ty, align), allocToken};
stackAddress = stackAddress.withAddress(
Builder.CreateElementBitCast(stackAddress.getAddress(), eltTy));
return stackAddress;
}
// Otherwise, use a dynamic alloca.
llvm::Value *stackRestorePoint = nullptr;
// Save the stack pointer if we are not in the entry block (we could be
// executed more than once).
bool isInEntryBlock = (Builder.GetInsertBlock() == &*CurFn->begin());
if (!isInEntryBlock) {
stackRestorePoint = Builder.CreateIntrinsicCall(
llvm::Intrinsic::stacksave,
{IGM.DataLayout.getAllocaPtrType(IGM.getLLVMContext())}, {}, "spsave");
}
// Emit the dynamic alloca.
auto *alloca = Builder.IRBuilderBase::CreateAlloca(eltTy, arraySize, name);
alloca->setAlignment(llvm::MaybeAlign(align.getValue()).valueOrOne());
assert(!isInEntryBlock ||
getActiveDominancePoint().isUniversal() &&
"Must be in entry block if we insert dynamic alloca's without "
"stackrestores");
return {Address(alloca, eltTy, align), stackRestorePoint};
}
/// Deallocate dynamic alloca's memory if requested by restoring the stack
/// location before the dynamic alloca's call.
void IRGenFunction::emitDeallocateDynamicAlloca(StackAddress address,
bool allowTaskDealloc,
bool useTaskDeallocThrough,
bool forCalleeCoroutineFrame) {
// Async function use taskDealloc.
if (allowTaskDealloc && isAsync() && address.getAddress().isValid()) {
if (useTaskDeallocThrough) {
emitTaskDeallocThrough(
Address(address.getExtraInfo(), IGM.Int8Ty, address.getAlignment()));
return;
}
emitTaskDealloc(
Address(address.getExtraInfo(), IGM.Int8Ty, address.getAlignment()));
return;
}
// In coroutines, unconditionally call llvm.coro.alloca.free.
// Except if the address is invalid, this happens when this is a StackAddress
// for a partial_apply [stack] that did not need a context object on the
// stack.
else if (isCoroutine() && address.getAddress().isValid()) {
// NOTE: llvm does not support dynamic allocas in coroutines.
auto allocToken = address.getExtraInfo();
if (!allocToken) {
#ifndef NDEBUG
auto *alloca = cast<llvm::AllocaInst>(address.getAddress().getAddress());
assert(isa<llvm::ConstantInt>(alloca->getArraySize()) &&
"Dynamic alloca without a token?!");
#endif
return;
}
Builder.CreateIntrinsicCall(forCalleeCoroutineFrame
? llvm::Intrinsic::coro_alloca_free_frame
: llvm::Intrinsic::coro_alloca_free,
allocToken);
return;
}
// Otherwise, call llvm.stackrestore if an address was saved.
auto savedSP = address.getExtraInfo();
if (savedSP == nullptr)
return;
Builder.CreateIntrinsicCall(llvm::Intrinsic::stackrestore,
{savedSP->getType()}, {savedSP});
}
/// Emit a call to do an 'initializeArrayWithCopy' operation.
void irgen::emitInitializeArrayWithCopyCall(IRGenFunction &IGF,
SILType T,
Address destObject,
Address srcObject,
llvm::Value *count) {
auto metadata = IGF.emitTypeMetadataRefForLayout(T);
auto dest = emitCastToOpaquePtr(IGF, destObject);
auto src = emitCastToOpaquePtr(IGF, srcObject);
IGF.Builder.CreateCall(IGF.IGM.getArrayInitWithCopyFunctionPointer(),
{dest, src, count, metadata});
}
/// Emit a call to do an 'initializeArrayWithTakeNoAlias' operation.
void irgen::emitInitializeArrayWithTakeNoAliasCall(IRGenFunction &IGF,
SILType T,
Address destObject,
Address srcObject,
llvm::Value *count) {
auto metadata = IGF.emitTypeMetadataRefForLayout(T);
auto dest = emitCastToOpaquePtr(IGF, destObject);
auto src = emitCastToOpaquePtr(IGF, srcObject);
IGF.Builder.CreateCall(IGF.IGM.getArrayInitWithTakeNoAliasFunctionPointer(),
{dest, src, count, metadata});
}
/// Emit a call to do an 'initializeArrayWithTakeFrontToBack' operation.
void irgen::emitInitializeArrayWithTakeFrontToBackCall(IRGenFunction &IGF,
SILType T,
Address destObject,
Address srcObject,
llvm::Value *count) {
auto metadata = IGF.emitTypeMetadataRefForLayout(T);
auto dest = emitCastToOpaquePtr(IGF, destObject);
auto src = emitCastToOpaquePtr(IGF, srcObject);
IGF.Builder.CreateCall(
IGF.IGM.getArrayInitWithTakeFrontToBackFunctionPointer(),
{dest, src, count, metadata});
}
/// Emit a call to do an 'initializeArrayWithTakeBackToFront' operation.
void irgen::emitInitializeArrayWithTakeBackToFrontCall(IRGenFunction &IGF,
SILType T,
Address destObject,
Address srcObject,
llvm::Value *count) {
auto metadata = IGF.emitTypeMetadataRefForLayout(T);
auto dest = emitCastToOpaquePtr(IGF, destObject);
auto src = emitCastToOpaquePtr(IGF, srcObject);
IGF.Builder.CreateCall(
IGF.IGM.getArrayInitWithTakeBackToFrontFunctionPointer(),
{dest, src, count, metadata});
}
/// Emit a call to do an 'assignWithCopy' operation.
void irgen::emitAssignWithCopyCall(IRGenFunction &IGF,
SILType T,
Address destObject,
Address srcObject) {
llvm::Value *metadata;
auto copyFn = IGF.emitValueWitnessFunctionRef(T, metadata,
ValueWitness::AssignWithCopy);
auto dest = emitCastToOpaquePtr(IGF, destObject);
auto src = emitCastToOpaquePtr(IGF, srcObject);
IGF.Builder.CreateCall(copyFn, {dest, src, metadata});
}
/// Emit a call to do an 'assignWithCopy' operation.
void irgen::emitAssignWithCopyCall(IRGenFunction &IGF,
llvm::Value *metadata,
Address destObject,
Address srcObject) {
auto copyFn = emitLoadOfValueWitnessFunctionFromMetadata(IGF, metadata,
ValueWitness::AssignWithCopy);
auto dest = emitCastToOpaquePtr(IGF, destObject);
auto src = emitCastToOpaquePtr(IGF, srcObject);
IGF.Builder.CreateCall(copyFn, {dest, src, metadata});
}
/// Emit a call to do an 'arrayAssignWithCopyNoAlias' operation.
void irgen::emitAssignArrayWithCopyNoAliasCall(IRGenFunction &IGF, SILType T,
Address destObject, Address srcObject,
llvm::Value *count) {
auto metadata = IGF.emitTypeMetadataRefForLayout(T);
auto dest = emitCastToOpaquePtr(IGF, destObject);
auto src = emitCastToOpaquePtr(IGF, srcObject);
IGF.Builder.CreateCall(IGF.IGM.getArrayAssignWithCopyNoAliasFunctionPointer(),
{dest, src, count, metadata});
}
/// Emit a call to do an 'arrayAssignWithCopyFrontToBack' operation.
void irgen::emitAssignArrayWithCopyFrontToBackCall(IRGenFunction &IGF,
SILType T,
Address destObject,
Address srcObject,
llvm::Value *count) {
auto metadata = IGF.emitTypeMetadataRefForLayout(T);
auto dest = emitCastToOpaquePtr(IGF, destObject);
auto src = emitCastToOpaquePtr(IGF, srcObject);
IGF.Builder.CreateCall(
IGF.IGM.getArrayAssignWithCopyFrontToBackFunctionPointer(),
{dest, src, count, metadata});
}
/// Emit a call to do an 'arrayAssignWithCopyBackToFront' operation.
void irgen::emitAssignArrayWithCopyBackToFrontCall(IRGenFunction &IGF,
SILType T,
Address destObject,
Address srcObject,
llvm::Value *count) {
auto metadata = IGF.emitTypeMetadataRefForLayout(T);
auto dest = emitCastToOpaquePtr(IGF, destObject);
auto src = emitCastToOpaquePtr(IGF, srcObject);
IGF.Builder.CreateCall(
IGF.IGM.getArrayAssignWithCopyBackToFrontFunctionPointer(),
{dest, src, count, metadata});
}
/// Emit a call to do an 'assignWithTake' operation.
void irgen::emitAssignWithTakeCall(IRGenFunction &IGF,
SILType T,
Address destObject,
Address srcObject) {
llvm::Value *metadata;
auto copyFn = IGF.emitValueWitnessFunctionRef(T, metadata,
ValueWitness::AssignWithTake);
auto dest = emitCastToOpaquePtr(IGF, destObject);
auto src = emitCastToOpaquePtr(IGF, srcObject);
IGF.Builder.CreateCall(copyFn, {dest, src, metadata});
}
/// Emit a call to do an 'arrayAssignWithTake' operation.
void irgen::emitAssignArrayWithTakeCall(IRGenFunction &IGF, SILType T,
Address destObject, Address srcObject,
llvm::Value *count) {
auto metadata = IGF.emitTypeMetadataRefForLayout(T);
auto dest = emitCastToOpaquePtr(IGF, destObject);
auto src = emitCastToOpaquePtr(IGF, srcObject);
IGF.Builder.CreateCall(IGF.IGM.getArrayAssignWithTakeFunctionPointer(),
{dest, src, count, metadata});
}
/// Emit a call to do a 'destroyArray' operation.
void irgen::emitDestroyArrayCall(IRGenFunction &IGF,
SILType T,
Address object,
llvm::Value *count) {
// If T is a trivial/POD type, nothing needs to be done.
if (IGF.IGM.getTypeProperties(T).isTrivial())
return;
auto metadata = IGF.emitTypeMetadataRefForLayout(T);
auto obj = emitCastToOpaquePtr(IGF, object);
IGF.Builder.CreateCall(IGF.IGM.getArrayDestroyFunctionPointer(),
{obj, count, metadata});
}
/// Emit a trampoline to call the getEnumTagSinglePayload witness. API:
/// UINT_TYPE (const T* enum, UINT_TYPE emptyCases, M *self)
static llvm::Constant *
getGetEnumTagSinglePayloadTrampolineFn(IRGenModule &IGM) {
llvm::Type *argTys[] = {IGM.OpaquePtrTy, IGM.Int32Ty, IGM.TypeMetadataPtrTy};
llvm::SmallString<40> fnName("__swift_getEnumTagSinglePayload");
auto func = IGM.getOrCreateHelperFunction(
fnName, IGM.Int32Ty, argTys,
[&](IRGenFunction &IGF) {
auto it = IGF.CurFn->arg_begin();
auto *enumAddr = &*(it++);
auto *numEmptyCases = &*(it++);
auto *metadata = &*(it++);
auto &Builder = IGF.Builder;
auto witnessFunc = emitLoadOfValueWitnessFunctionFromMetadata(
IGF, metadata, ValueWitness::GetEnumTagSinglePayload);
auto *result = Builder.CreateCall(witnessFunc,
{enumAddr, numEmptyCases, metadata});
Builder.CreateRet(result);
},
true /*noinline*/);
// This function is readonly.
cast<llvm::Function>(func)->setOnlyReadsMemory();
return func;
}
/// Emit a trampoline to call the storeEnumTagSinglePayload witness. API:
/// VOID_TYPE (const T* enum, UINT_TYPE whichCase, UINT_TYPE emptyCases,
/// M *self)
static llvm::Constant *
getStoreEnumTagSinglePayloadTrampolineFn(IRGenModule &IGM) {
llvm::Type *argTys[] = {IGM.OpaquePtrTy, IGM.Int32Ty, IGM.Int32Ty,
IGM.TypeMetadataPtrTy};
llvm::SmallString<40> fnName("__swift_storeEnumTagSinglePayload");
return IGM.getOrCreateHelperFunction(
fnName, IGM.VoidTy, argTys,
[&](IRGenFunction &IGF) {
auto it = IGF.CurFn->arg_begin();
auto *enumAddr = &*(it++);
auto *whichCase = &*(it++);
auto *numEmptyCases = &*(it++);
auto *metadata = &*(it++);
auto &Builder = IGF.Builder;
auto witnessFunc = emitLoadOfValueWitnessFunctionFromMetadata(
IGF, metadata, ValueWitness::StoreEnumTagSinglePayload);
Builder.CreateCall(witnessFunc,
{enumAddr, whichCase, numEmptyCases, metadata});
Builder.CreateRetVoid();
},
true /*noinline*/);
}
llvm::Value *irgen::emitGetEnumTagSinglePayloadCall(IRGenFunction &IGF,
SILType T,
llvm::Value *numEmptyCases,
Address destObject) {
if (!IGF.optimizeForSize()) {
llvm::Value *metadata;
auto fn = IGF.emitValueWitnessFunctionRef(
T, metadata, ValueWitness::GetEnumTagSinglePayload);
auto dest = emitCastToOpaquePtr(IGF, destObject);
llvm::CallInst *call = IGF.Builder.CreateCall(
fn, {dest, numEmptyCases, metadata});
return call;
}
auto *metadata = IGF.emitTypeMetadataRefForLayout(T);
auto *func = getGetEnumTagSinglePayloadTrampolineFn(IGF.IGM);
auto *fnType = cast<llvm::Function>(func)->getFunctionType();
auto dest = emitCastToOpaquePtr(IGF, destObject);
auto *result =
IGF.Builder.CreateCall(fnType, func, {dest, numEmptyCases, metadata});
return result;
}
void irgen::emitStoreEnumTagSinglePayloadCall(
IRGenFunction &IGF, SILType T, llvm::Value *whichCase,
llvm::Value *numEmptyCases, Address destObject) {
if (!IGF.optimizeForSize()) {
llvm::Value *metadata;
auto fn = IGF.emitValueWitnessFunctionRef(
T, metadata, ValueWitness::StoreEnumTagSinglePayload);
auto dest = emitCastToOpaquePtr(IGF, destObject);
IGF.Builder.CreateCall(fn, {dest, whichCase, numEmptyCases, metadata});
return;
}
auto *metadata = IGF.emitTypeMetadataRefForLayout(T);
auto *func = getStoreEnumTagSinglePayloadTrampolineFn(IGF.IGM);
auto *fnType = cast<llvm::Function>(func)->getFunctionType();
auto dest = emitCastToOpaquePtr(IGF, destObject);
IGF.Builder.CreateCall(fnType, func,
{dest, whichCase, numEmptyCases, metadata});
}
/// Emit a call to the 'getEnumTag' operation.
llvm::Value *irgen::emitGetEnumTagCall(IRGenFunction &IGF,
SILType T,
Address srcObject) {
llvm::Value *metadata;
auto fn = IGF.emitValueWitnessFunctionRef(T, metadata,
ValueWitness::GetEnumTag);
auto src = emitCastToOpaquePtr(IGF, srcObject);
llvm::CallInst *call =
IGF.Builder.CreateCall(fn, {src, metadata});
return call;
}
/// Emit a call to the 'destructiveProjectEnumData' operation.
/// The type must be dynamically known to have enum witnesses.
void irgen::emitDestructiveProjectEnumDataCall(IRGenFunction &IGF,
SILType T,
Address srcObject) {
llvm::Value *metadata;
auto fn = IGF.emitValueWitnessFunctionRef(T, metadata,
ValueWitness::DestructiveProjectEnumData);
auto src = emitCastToOpaquePtr(IGF, srcObject);
IGF.Builder.CreateCall(fn, {src, metadata});
}
/// Emit a call to the 'destructiveInjectEnumTag' operation.
/// The type must be dynamically known to have enum witnesses.
void irgen::emitDestructiveInjectEnumTagCall(IRGenFunction &IGF,
SILType T,
llvm::Value *tagValue,
Address srcObject) {
llvm::Value *metadata;
auto fn = IGF.emitValueWitnessFunctionRef(T, metadata,
ValueWitness::DestructiveInjectEnumTag);
auto src = emitCastToOpaquePtr(IGF, srcObject);
IGF.Builder.CreateCall(fn, {src, tagValue, metadata});
}
/// Load the 'size' value witness from the given table as a size_t.
llvm::Value *irgen::emitLoadOfSize(IRGenFunction &IGF, SILType T) {
return IGF.emitValueWitnessValue(T, ValueWitness::Size);
}
/// Load the 'alignmentMask' value witness from the given table as a size_t.
llvm::Value *irgen::emitLoadOfAlignmentMask(IRGenFunction &IGF, SILType T) {
auto flags = IGF.emitValueWitnessValue(T, ValueWitness::Flags);
return emitAlignMaskFromFlags(IGF, flags);
}
/// Load the 'isTriviallyDestroyable' valueWitness from the given table as an i1.
llvm::Value *irgen::emitLoadOfIsTriviallyDestroyable(IRGenFunction &IGF, SILType T) {
auto flags = IGF.emitValueWitnessValue(T, ValueWitness::Flags);
auto mask = IGF.IGM.getInt32(ValueWitnessFlags::IsNonPOD);
auto masked = IGF.Builder.CreateAnd(flags, mask);
return IGF.Builder.CreateICmpEQ(masked, IGF.IGM.getInt32(0),
flags->getName() + ".isTriviallyDestroyable");
}
/// Load the 'isBitwiseTakable' valueWitness from the given table as an i1.
llvm::Value *irgen::emitLoadOfIsBitwiseTakable(IRGenFunction &IGF, SILType T) {
auto flags = IGF.emitValueWitnessValue(T, ValueWitness::Flags);
auto mask = IGF.IGM.getInt32(ValueWitnessFlags::IsNonBitwiseTakable);
auto masked = IGF.Builder.CreateAnd(flags, mask);
return IGF.Builder.CreateICmpEQ(masked, IGF.IGM.getInt32(0),
flags->getName() + ".isBitwiseTakable");
}
/// Load the 'isInline' valueWitness from the given table as an i1.
llvm::Value *irgen::emitLoadOfIsInline(IRGenFunction &IGF, SILType T) {
auto flags = IGF.emitValueWitnessValue(T, ValueWitness::Flags);
auto mask = IGF.IGM.getInt32(ValueWitnessFlags::IsNonInline);
auto masked = IGF.Builder.CreateAnd(flags, mask);
return IGF.Builder.CreateICmpEQ(masked, IGF.IGM.getInt32(0),
flags->getName() + ".isInline");
}
/// Load the 'stride' value witness from the given table as a size_t.
llvm::Value *irgen::emitLoadOfStride(IRGenFunction &IGF, SILType T) {
return IGF.emitValueWitnessValue(T, ValueWitness::Stride);
}
llvm::Value *irgen::emitLoadOfExtraInhabitantCount(IRGenFunction &IGF,
SILType T) {
return IGF.emitValueWitnessValue(T, ValueWitness::ExtraInhabitantCount);
}
void irgen::emitStoreOfExtraInhabitantCount(IRGenFunction &IGF,
llvm::Value *value,
llvm::Value *metadata) {
auto vwTable = IGF.emitValueWitnessTableRefForMetadata(metadata);
auto addr = emitAddressOfValueWitnessTableValue(
IGF, vwTable, ValueWitness::ExtraInhabitantCount);
IGF.Builder.CreateStore(value, addr);
}
std::pair<llvm::Value *, llvm::Value *>
irgen::emitLoadOfIsInline(IRGenFunction &IGF, llvm::Value *metadata) {
auto *flags = emitLoadOfValueWitnessValueFromMetadata(IGF, metadata,
ValueWitness::Flags);
auto mask = IGF.IGM.getInt32(ValueWitnessFlags::IsNonInline);
auto masked = IGF.Builder.CreateAnd(flags, mask);
return std::make_pair(
IGF.Builder.CreateICmpEQ(masked, IGF.IGM.getInt32(0),
flags->getName() + ".isInline"),
flags);
}
llvm::Value *irgen::emitLoadOfSize(IRGenFunction &IGF, llvm::Value *metadata) {
auto *size = emitLoadOfValueWitnessValueFromMetadata(IGF, metadata,
ValueWitness::Size);
return size;
}
llvm::Value *irgen::emitAlignMaskFromFlags(IRGenFunction &IGF,
llvm::Value *flags) {
auto flagsAsSize = IGF.Builder.CreateZExtOrTrunc(flags, IGF.IGM.SizeTy);
auto *alignMask = IGF.IGM.getSize(Size(ValueWitnessFlags::AlignmentMask));
return IGF.Builder.CreateAnd(flagsAsSize, alignMask,
flags->getName() + ".alignmentMask");
}
/// Emit a call to do an 'initializeWithCopy' operation.
void irgen::emitInitializeWithCopyCall(IRGenFunction &IGF,
SILType T,
Address dest,
Address src) {
llvm::Value *metadata;
auto fn = IGF.emitValueWitnessFunctionRef(T, metadata,
ValueWitness::InitializeWithCopy);
auto destPtr = emitCastToOpaquePtr(IGF, dest);
auto srcPtr = emitCastToOpaquePtr(IGF, src);
IGF.Builder.CreateCall(fn, {destPtr, srcPtr, metadata});
}
llvm::Value *irgen::emitInitializeWithCopyCall(IRGenFunction &IGF,
llvm::Value *metadata,
Address dest, Address src) {
auto copyFn = emitLoadOfValueWitnessFunctionFromMetadata(
IGF, metadata, ValueWitness::InitializeWithCopy);
auto destPtr = emitCastToOpaquePtr(IGF, dest);
auto srcPtr = emitCastToOpaquePtr(IGF, src);
llvm::CallInst *call = IGF.Builder.CreateCall(
copyFn, {destPtr, srcPtr, metadata});
return call;
}
/// Emit a call to do an 'initializeWithTake' operation.
void irgen::emitInitializeWithTakeCall(IRGenFunction &IGF,
SILType T,
Address dest,
Address src) {
llvm::Value *metadata;
auto fn = IGF.emitValueWitnessFunctionRef(T, metadata,
ValueWitness::InitializeWithTake);
auto destPtr = emitCastToOpaquePtr(IGF, dest);
auto srcPtr = emitCastToOpaquePtr(IGF, src);
IGF.Builder.CreateCall(fn, {destPtr, srcPtr, metadata});
}
llvm::Value *irgen::emitInitializeWithTakeCall(IRGenFunction &IGF,
llvm::Value *metadata,
Address dest, Address src) {
auto copyFn = emitLoadOfValueWitnessFunctionFromMetadata(
IGF, metadata, ValueWitness::InitializeWithTake);
auto destPtr = emitCastToOpaquePtr(IGF, dest);
auto srcPtr = emitCastToOpaquePtr(IGF, src);
llvm::CallInst *call =
IGF.Builder.CreateCall(copyFn, {destPtr, srcPtr, metadata});
return call;
}
/// Emit a call to do a 'destroy' operation.
void irgen::emitDestroyCall(IRGenFunction &IGF,
SILType T,
Address object) {
// If T is a trivial/POD type, nothing needs to be done.
if (IGF.IGM.getTypeProperties(T).isTrivial())
return;
llvm::Value *metadata;
auto fn = IGF.emitValueWitnessFunctionRef(T, metadata,
ValueWitness::Destroy);
auto objectPtr = emitCastToOpaquePtr(IGF, object);
IGF.Builder.CreateCall(fn, {objectPtr, metadata});
}
void irgen::emitDestroyCall(IRGenFunction &IGF, llvm::Value *metadata,
Address object) {
auto fn = emitLoadOfValueWitnessFunctionFromMetadata(IGF, metadata,
ValueWitness::Destroy);
auto objectPtr = emitCastToOpaquePtr(IGF, object);
IGF.Builder.CreateCall(fn, {objectPtr, metadata});
}
static llvm::Constant *getAllocateValueBufferFunction(IRGenModule &IGM) {
llvm::Type *argTys[] = {IGM.TypeMetadataPtrTy, IGM.OpaquePtrTy};
llvm::SmallString<40> fnName("__swift_allocate_value_buffer");
return IGM.getOrCreateHelperFunction(
fnName, IGM.OpaquePtrTy, argTys,
[&](IRGenFunction &IGF) {
auto it = IGF.CurFn->arg_begin();
auto *metadata = &*(it++);
auto buffer = Address(&*(it++), IGM.OpaqueTy, Alignment(1));
// Dynamically check whether this type is inline or needs an allocation.
llvm::Value *isInline, *flags;
std::tie(isInline, flags) = emitLoadOfIsInline(IGF, metadata);
auto *outlineBB = IGF.createBasicBlock("outline.allocateValueInBuffer");
auto *doneBB = IGF.createBasicBlock("done");
llvm::Value *addressInline, *addressOutline;
addressInline = buffer.getAddress();
auto *origBB = IGF.Builder.GetInsertBlock();
IGF.Builder.CreateCondBr(isInline, doneBB, outlineBB);
IGF.Builder.emitBlock(outlineBB);
{
auto *size = emitLoadOfSize(IGF, metadata);
auto *alignMask = emitAlignMaskFromFlags(IGF, flags);
auto valueAddr =
IGF.emitAllocRawCall(size, alignMask, "outline.ValueBuffer");
IGF.Builder.CreateStore(
valueAddr,
Address(IGF.Builder.CreateBitCast(buffer.getAddress(), IGM.PtrTy),
IGM.Int8PtrTy, Alignment(1)));
addressOutline =
IGF.Builder.CreateBitCast(valueAddr, IGM.OpaquePtrTy);
IGF.Builder.CreateBr(doneBB);
}
IGF.Builder.emitBlock(doneBB);
auto *addressOfValue = IGF.Builder.CreatePHI(IGM.OpaquePtrTy, 2);
addressOfValue->addIncoming(addressInline, origBB);
addressOfValue->addIncoming(addressOutline, outlineBB);
IGF.Builder.CreateRet(addressOfValue);
},
true /*noinline*/);
}
Address irgen::emitAllocateValueInBuffer(IRGenFunction &IGF, SILType type,
Address buffer) {
// Handle FixedSize types.
auto &IGM = IGF.IGM;
auto *storagePtrTy = IGM.PtrTy;
auto storageTy = IGM.getStorageType(type);
auto &Builder = IGF.Builder;
if (auto *fixedTI = dyn_cast<FixedTypeInfo>(&IGF.getTypeInfo(type))) {
auto packing = fixedTI->getFixedPacking(IGM);
// Inline representation.
if (packing == FixedPacking::OffsetZero) {
return Address(Builder.CreateBitCast(buffer.getAddress(), storagePtrTy),
storageTy, buffer.getAlignment());
}
// Outline representation.
assert(packing == FixedPacking::Allocate && "Expect non dynamic packing");
auto size = fixedTI->getStaticSize(IGM);
auto alignMask = fixedTI->getStaticAlignmentMask(IGM);
auto valueAddr =
IGF.emitAllocRawCall(size, alignMask, "outline.ValueBuffer");
Builder.CreateStore(valueAddr,
Address(Builder.CreateBitCast(buffer.getAddress(),
IGF.IGM.Int8PtrPtrTy),
IGM.Int8PtrTy, buffer.getAlignment()));
return Address(Builder.CreateBitCast(valueAddr, storagePtrTy), storageTy,
buffer.getAlignment());
}
// Dynamic packing.
/// Call a function to handle the non-fixed case.
auto *allocateFun = getAllocateValueBufferFunction(IGF.IGM);
auto *fnType = cast<llvm::Function>(allocateFun)->getFunctionType();
auto *metadata = IGF.emitTypeMetadataRefForLayout(type);
auto *call = Builder.CreateCall(
fnType, allocateFun,
{metadata, Builder.CreateBitCast(buffer.getAddress(), IGM.OpaquePtrTy)});
call->setCallingConv(IGF.IGM.DefaultCC);
call->setDoesNotThrow();
auto addressOfValue = Builder.CreateBitCast(call, storagePtrTy);
return Address(addressOfValue, storageTy, Alignment(1));
}
static llvm::Constant *getProjectValueInBufferFunction(IRGenModule &IGM) {
llvm::Type *argTys[] = {IGM.TypeMetadataPtrTy, IGM.OpaquePtrTy};
llvm::SmallString<40> fnName("__swift_project_value_buffer");
return IGM.getOrCreateHelperFunction(
fnName, IGM.OpaquePtrTy, argTys,
[&](IRGenFunction &IGF) {
auto it = IGF.CurFn->arg_begin();
auto *metadata = &*(it++);
auto buffer = Address(&*(it++), IGM.OpaqueTy, Alignment(1));
auto &Builder = IGF.Builder;
// Dynamically check whether this type is inline or needs an allocation.
llvm::Value *isInline, *flags;
std::tie(isInline, flags) = emitLoadOfIsInline(IGF, metadata);
auto *outlineBB = IGF.createBasicBlock("outline.projectValueInBuffer");
auto *doneBB = IGF.createBasicBlock("done");
llvm::Value *addressInline, *addressOutline;
auto *origBB = Builder.GetInsertBlock();
addressInline = buffer.getAddress();
Builder.CreateCondBr(isInline, doneBB, outlineBB);
Builder.emitBlock(outlineBB);
{
addressOutline = Builder.CreateLoad(
Address(Builder.CreateBitCast(buffer.getAddress(), IGM.PtrTy),
IGM.OpaquePtrTy, Alignment(1)));
Builder.CreateBr(doneBB);
}
Builder.emitBlock(doneBB);
auto *addressOfValue = Builder.CreatePHI(IGM.OpaquePtrTy, 2);
addressOfValue->addIncoming(addressInline, origBB);
addressOfValue->addIncoming(addressOutline, outlineBB);
Builder.CreateRet(addressOfValue);
},
true /*noinline*/);
}
Address irgen::emitProjectValueInBuffer(IRGenFunction &IGF, SILType type,
Address buffer) {
// Handle FixedSize types.
auto &IGM = IGF.IGM;
auto *storagePtrTy = IGM.PtrTy;
auto storageTy = IGM.getStorageType(type);
auto &Builder = IGF.Builder;
if (auto *fixedTI = dyn_cast<FixedTypeInfo>(&IGF.getTypeInfo(type))) {
auto packing = fixedTI->getFixedPacking(IGM);
// Inline representation.
if (packing == FixedPacking::OffsetZero) {
return Address(Builder.CreateBitCast(buffer.getAddress(), storagePtrTy),
storageTy, buffer.getAlignment());
}
// Outline representation.
assert(packing == FixedPacking::Allocate && "Expect non dynamic packing");
auto valueAddr = Builder.CreateLoad(
Address(Builder.CreateBitCast(buffer.getAddress(), IGM.PtrTy),
storagePtrTy, buffer.getAlignment()));
return Address(Builder.CreateBitCast(valueAddr, storagePtrTy), storageTy,
buffer.getAlignment());
}
// Dynamic packing.
auto *projectFun = getProjectValueInBufferFunction(IGF.IGM);
auto *fnType = cast<llvm::Function>(projectFun)->getFunctionType();
auto *metadata = IGF.emitTypeMetadataRefForLayout(type);
auto *call = Builder.CreateCall(
fnType, projectFun,
{metadata, Builder.CreateBitCast(buffer.getAddress(), IGM.OpaquePtrTy)});
call->setCallingConv(IGF.IGM.DefaultCC);
call->setDoesNotThrow();
auto addressOfValue = Builder.CreateBitCast(call, storagePtrTy);
return Address(addressOfValue, storageTy, Alignment(1));
}
llvm::Value *
irgen::emitGetEnumTagSinglePayloadGenericCall(IRGenFunction &IGF,
SILType payloadType,
const TypeInfo &payloadTI,
llvm::Value *numExtraCases,
Address address,
GetExtraInhabitantTagEmitter emitter) {
auto getExtraInhabitantTagFn =
getOrCreateGetExtraInhabitantTagFunction(IGF.IGM, payloadType,
payloadTI, emitter);
// Sign the getExtraInhabitantTag function with the C function pointer schema.
if (auto schema = IGF.IGM.getOptions().PointerAuth.FunctionPointers) {
if (schema.hasOtherDiscrimination())
schema = IGF.IGM.getOptions().PointerAuth.GetExtraInhabitantTagFunction;
getExtraInhabitantTagFn = IGF.IGM.getConstantSignedPointer(
getExtraInhabitantTagFn, schema, PointerAuthEntity(), nullptr);
}
// We assume this is never a reabstracted type.
auto type = payloadType.getASTType();
assert(type->isLegalFormalType());
auto metadata = IGF.emitTypeMetadataRef(type);
auto ptr = IGF.Builder.CreateBitCast(address.getAddress(),
IGF.IGM.OpaquePtrTy);
auto getEnumTagGenericFn =
IGF.IGM.getGetEnumTagSinglePayloadGenericFunctionPointer();
auto call = IGF.Builder.CreateCall(getEnumTagGenericFn,
{ptr,
numExtraCases,
metadata,
getExtraInhabitantTagFn});
call->setCallingConv(IGF.IGM.SwiftCC);
return call;
}
llvm::Constant *
irgen::getOrCreateGetExtraInhabitantTagFunction(IRGenModule &IGM,
SILType objectType,
const TypeInfo &objectTI,
GetExtraInhabitantTagEmitter emitter) {
// We assume this is never a reabstracted type.
CanType type = objectType.getASTType();
assert(type->isLegalFormalType());
auto fnTy = llvm::FunctionType::get(IGM.Int32Ty,
{IGM.OpaquePtrTy,
IGM.Int32Ty,
IGM.TypeMetadataPtrTy},
false);
// TODO: use a meaningful mangled name and internal/shared linkage.
auto fn = llvm::Function::Create(fnTy, llvm::Function::PrivateLinkage,
"__swift_get_extra_inhabitant_index",
&IGM.Module);
fn->setAttributes(IGM.constructInitialAttributes());
fn->setCallingConv(IGM.SwiftCC);
IRGenFunction IGF(IGM, fn);
if (IGM.DebugInfo)
IGM.DebugInfo->emitArtificialFunction(IGF, fn);
auto parameters = IGF.collectParameters();
auto ptr = parameters.claimNext();
auto xiCount = parameters.claimNext();
auto metadata = parameters.claimNext();
// Bind the metadata to make any archetypes available.
IGF.bindLocalTypeDataFromTypeMetadata(type, IsExact, metadata,
MetadataState::Complete);
// Form a well-typed address from the opaque pointer.
ptr = IGF.Builder.CreateBitCast(ptr, IGM.PtrTy);
Address addr = objectTI.getAddressForPointer(ptr);
auto tag = emitter(IGF, addr, xiCount);
IGF.Builder.CreateRet(tag);
return fn;
}
void
irgen::emitStoreEnumTagSinglePayloadGenericCall(IRGenFunction &IGF,
SILType payloadType,
const TypeInfo &payloadTI,
llvm::Value *whichCase,
llvm::Value *numExtraCases,
Address address,
StoreExtraInhabitantTagEmitter emitter) {
auto storeExtraInhabitantTagFn =
getOrCreateStoreExtraInhabitantTagFunction(IGF.IGM, payloadType,
payloadTI, emitter);
// Sign the getExtraInhabitantTag function with the C function pointer schema.
if (auto schema = IGF.IGM.getOptions().PointerAuth.FunctionPointers) {
if (schema.hasOtherDiscrimination())
schema = IGF.IGM.getOptions().PointerAuth.StoreExtraInhabitantTagFunction;
storeExtraInhabitantTagFn = IGF.IGM.getConstantSignedPointer(
storeExtraInhabitantTagFn, schema, PointerAuthEntity(), nullptr);
}
// We assume this is never a reabstracted type.
auto type = payloadType.getASTType();
assert(type->isLegalFormalType());
auto metadata = IGF.emitTypeMetadataRef(type);
auto ptr = IGF.Builder.CreateBitCast(address.getAddress(),
IGF.IGM.OpaquePtrTy);
auto storeEnumTagGenericFn =
IGF.IGM.getStoreEnumTagSinglePayloadGenericFunctionPointer();
auto call = IGF.Builder.CreateCall(storeEnumTagGenericFn,
{ptr,
whichCase,
numExtraCases,
metadata,
storeExtraInhabitantTagFn});
call->setCallingConv(IGF.IGM.SwiftCC);
}
llvm::Constant *
irgen::getOrCreateStoreExtraInhabitantTagFunction(IRGenModule &IGM,
SILType objectType,
const TypeInfo &objectTI,
StoreExtraInhabitantTagEmitter emitter) {
// We assume this is never a reabstracted type.
CanType type = objectType.getASTType();
assert(type->isLegalFormalType());
auto fnTy = llvm::FunctionType::get(IGM.VoidTy,
{IGM.OpaquePtrTy,
IGM.Int32Ty,
IGM.Int32Ty,
IGM.TypeMetadataPtrTy},
false);
// TODO: use a meaningful mangled name and internal/shared linkage.
auto fn = llvm::Function::Create(fnTy, llvm::Function::PrivateLinkage,
"__swift_store_extra_inhabitant_index",
&IGM.Module);
fn->setAttributes(IGM.constructInitialAttributes());
fn->setCallingConv(IGM.SwiftCC);
IRGenFunction IGF(IGM, fn);
if (IGM.DebugInfo)
IGM.DebugInfo->emitArtificialFunction(IGF, fn);
auto parameters = IGF.collectParameters();
auto ptr = parameters.claimNext();
auto tag = parameters.claimNext();
auto xiCount = parameters.claimNext();
auto metadata = parameters.claimNext();
// Bind the metadata to make any archetypes available.
IGF.bindLocalTypeDataFromTypeMetadata(type, IsExact, metadata,
MetadataState::Complete);
// Form a well-typed address from the opaque pointer.
ptr = IGF.Builder.CreateBitCast(ptr, IGM.PtrTy);
Address addr = objectTI.getAddressForPointer(ptr);
emitter(IGF, addr, tag, xiCount);
IGF.Builder.CreateRetVoid();
return fn;
}
llvm::Value *TypeInfo::getExtraInhabitantTagDynamic(IRGenFunction &IGF,
Address address,
SILType T,
llvm::Value *knownXICount,
bool isOutlined) const {
if (auto fixedTI = dyn_cast<FixedTypeInfo>(this)) {
auto index = fixedTI->getExtraInhabitantIndex(IGF, address, T, isOutlined);
// The runtime APIs expect that 0 means the payload case and 1+
// means the extra cases, but in IRGen, getExtraInhabitantIndex
// returns -1 for the payload case and 0+ for extra inhabitants.
// This is an easy adjustment to make.
auto one = llvm::ConstantInt::get(IGF.IGM.Int32Ty, 1);
auto tag = IGF.Builder.CreateAdd(index, one);
return tag;
} else {
if (!knownXICount)
knownXICount = emitLoadOfExtraInhabitantCount(IGF, T);
auto tag = getEnumTagSinglePayload(IGF, /*num extra cases*/ knownXICount,
address, T, isOutlined);
return tag;
}
}
void TypeInfo::storeExtraInhabitantTagDynamic(IRGenFunction &IGF,
llvm::Value *tag,
Address address,
SILType T,
bool isOutlined) const {
if (auto fixedTI = dyn_cast<FixedTypeInfo>(this)) {
// The runtime APIs expect that 0 means the payload case and 1+
// means the extra cases, but in IRGen, storeExtraInhabitant
// expects extra inhabitants to be indexed as 0+.
// This is an easy adjustment to make.
auto one = llvm::ConstantInt::get(IGF.IGM.Int32Ty, 1);
auto index = IGF.Builder.CreateSub(tag, one);
fixedTI->storeExtraInhabitant(IGF, index, address, T, isOutlined);
} else {
storeEnumTagSinglePayload(IGF, tag, tag, address, T, isOutlined);
}
}
Reference in New Issue View Git Blame Copy Permalink