mirror of
https://github.com/apple/swift.git
synced 2026-06-20 15:42:51 +02:00
Kavon Farvardin
b7d57f3bcd
Ordinary base protocols use fixed-indexing to access the base index. That means adding another base protocol to an existing protocol can break the order of the entries, and thus clients, because we otherwise order the base entires with TypeDecl::compare. Reparentable protocols are meant to be resilient to that, so we order them at the end of the base entries list, just before the other resilient entries in the witness table. This patch completes the picture, by having the reparentable protocol entries be indexed resiliently, in the same manner as associated conformances. The difference is that we can skip the call to `swift_getAssociatedConformanceWitness` and compute the index directly by finding the distance of the descriptors, because we know all base protocol witness table entries are eagarly instantiated. Using this distance protects us from the ordering problems of entries among all of the reparentable base protocols. resolves rdar://173409851
1771 lines
71 KiB
C++
1771 lines
71 KiB
C++
//===--- GenOpaque.cpp - Swift IR-generation for opaque values ------------===//
|
|
//
|
|
// This source file is part of the Swift.org open source project
|
|
//
|
|
// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
|
|
// Licensed under Apache License v2.0 with Runtime Library Exception
|
|
//
|
|
// See https://swift.org/LICENSE.txt for license information
|
|
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
//
|
|
// This file implements IR generation for opaque values and value
|
|
// witness operations.
|
|
//
|
|
// In the comments throughout this file, three type names are used:
|
|
// 'B' is the type of a fixed-size buffer
|
|
// 'T' is the type which implements a protocol
|
|
// 'W' is the type of a witness to the protocol
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#include "llvm/ADT/SmallString.h"
|
|
#include "llvm/Support/raw_ostream.h"
|
|
#include "llvm/IR/DerivedTypes.h"
|
|
#include "swift/AST/IRGenOptions.h"
|
|
#include "swift/ABI/MetadataValues.h"
|
|
#include "swift/Basic/Assertions.h"
|
|
#include "swift/IRGen/ValueWitness.h"
|
|
#include "swift/SIL/TypeLowering.h"
|
|
|
|
#include "Callee.h"
|
|
#include "ComputedWitnessIndex.h"
|
|
#include "Explosion.h"
|
|
#include "FixedTypeInfo.h"
|
|
#include "GenPointerAuth.h"
|
|
#include "IRGenDebugInfo.h"
|
|
#include "IRGenFunction.h"
|
|
#include "IRGenModule.h"
|
|
#include "ProtocolInfo.h"
|
|
|
|
#include "GenOpaque.h"
|
|
|
|
using namespace swift;
|
|
using namespace irgen;
|
|
|
|
/// A fixed-size buffer is always three pointers in size and pointer-aligned.
|
|
/// If we align them more, we'll need to introduce padding to
|
|
/// make protocol types work.
|
|
Size irgen::getFixedBufferSize(IRGenModule &IGM) {
|
|
return NumWords_ValueBuffer * IGM.getPointerSize();
|
|
}
|
|
Alignment irgen::getFixedBufferAlignment(IRGenModule &IGM) {
|
|
return IGM.getPointerAlignment();
|
|
}
|
|
|
|
/// Lazily create the standard fixed-buffer type.
|
|
llvm::Type *IRGenModule::getFixedBufferTy() {
|
|
if (FixedBufferTy) return FixedBufferTy;
|
|
|
|
auto size = getFixedBufferSize(*this).getValue();
|
|
FixedBufferTy = llvm::ArrayType::get(Int8Ty, size);
|
|
return FixedBufferTy;
|
|
}
|
|
|
|
static llvm::Type *createWitnessType(IRGenModule &IGM, ValueWitness index) {
|
|
switch (index) {
|
|
// void (*destroy)(T *object, witness_t *self);
|
|
case ValueWitness::Destroy: {
|
|
llvm::Type *args[] = { IGM.OpaquePtrTy, IGM.TypeMetadataPtrTy };
|
|
return llvm::FunctionType::get(IGM.VoidTy, args, /*isVarArg*/ false);
|
|
}
|
|
|
|
// T *(*initializeBufferWithCopyOfBuffer)(B *dest, B *src, M *self);
|
|
case ValueWitness::InitializeBufferWithCopyOfBuffer: {
|
|
auto *bufPtrTy = IGM.PtrTy;
|
|
llvm::Type *args[] = { bufPtrTy, bufPtrTy, IGM.TypeMetadataPtrTy };
|
|
return llvm::FunctionType::get(IGM.OpaquePtrTy, args, /*isVarArg*/ false);
|
|
}
|
|
|
|
// T *(*assignWithCopy)(T *dest, T *src, M *self);
|
|
// T *(*assignWithTake)(T *dest, T *src, M *self);
|
|
// T *(*initializeWithCopy)(T *dest, T *src, M *self);
|
|
// T *(*initializeWithTake)(T *dest, T *src, M *self);
|
|
case ValueWitness::AssignWithCopy:
|
|
case ValueWitness::AssignWithTake:
|
|
case ValueWitness::InitializeWithCopy:
|
|
case ValueWitness::InitializeWithTake: {
|
|
llvm::Type *ptrTy = IGM.OpaquePtrTy;
|
|
llvm::Type *args[] = { ptrTy, ptrTy, IGM.TypeMetadataPtrTy };
|
|
return llvm::FunctionType::get(ptrTy, args, /*isVarArg*/ false);
|
|
}
|
|
|
|
/// unsigned (*getEnumTag)(T *obj, M *self);
|
|
case ValueWitness::GetEnumTag: {
|
|
llvm::Type *ptrTy = IGM.OpaquePtrTy;
|
|
llvm::Type *metaTy = IGM.TypeMetadataPtrTy;
|
|
llvm::Type *indexTy = IGM.Int32Ty;
|
|
|
|
llvm::Type *args[] = {ptrTy, metaTy};
|
|
|
|
return llvm::FunctionType::get(indexTy, args, /*isVarArg*/ false);
|
|
}
|
|
|
|
/// void (*destructiveProjectEnumData)(T *obj, M *self);
|
|
case ValueWitness::DestructiveProjectEnumData: {
|
|
llvm::Type *voidTy = IGM.VoidTy;
|
|
llvm::Type *ptrTy = IGM.OpaquePtrTy;
|
|
llvm::Type *metaTy = IGM.TypeMetadataPtrTy;
|
|
|
|
llvm::Type *args[] = {ptrTy, metaTy};
|
|
|
|
return llvm::FunctionType::get(voidTy, args, /*isVarArg*/ false);
|
|
}
|
|
|
|
/// void (*destructiveInjectEnumTag)(T *obj, unsigned tag, M *self);
|
|
case ValueWitness::DestructiveInjectEnumTag: {
|
|
llvm::Type *voidTy = IGM.VoidTy;
|
|
llvm::Type *ptrTy = IGM.OpaquePtrTy;
|
|
llvm::Type *indexTy = IGM.Int32Ty;
|
|
llvm::Type *metaTy = IGM.TypeMetadataPtrTy;
|
|
|
|
llvm::Type *args[] = {ptrTy, indexTy, metaTy};
|
|
|
|
return llvm::FunctionType::get(voidTy, args, /*isVarArg*/ false);
|
|
}
|
|
|
|
/// unsigned (*getEnumTagSinglePayload)(const T* enum, UINT_TYPE emptyCases,
|
|
/// M *self)
|
|
case ValueWitness::GetEnumTagSinglePayload: {
|
|
llvm::Type *ptrTy = IGM.OpaquePtrTy;
|
|
llvm::Type *indexTy = IGM.Int32Ty;
|
|
llvm::Type *metaTy = IGM.TypeMetadataPtrTy;
|
|
|
|
llvm::Type *args[] = { ptrTy, indexTy, metaTy };
|
|
return llvm::FunctionType::get(indexTy, args, false);
|
|
}
|
|
|
|
/// void (*storeEnumTagSinglePayload)(T* enum, UINT_TYPE whichCase,
|
|
/// UINT_TYPE emptyCases,
|
|
/// M *self)
|
|
case ValueWitness::StoreEnumTagSinglePayload: {
|
|
llvm::Type *voidTy = IGM.VoidTy;
|
|
llvm::Type *ptrTy = IGM.OpaquePtrTy;
|
|
llvm::Type *indexTy = IGM.Int32Ty;
|
|
llvm::Type *metaTy = IGM.TypeMetadataPtrTy;
|
|
|
|
llvm::Type *args[] = { ptrTy, indexTy, indexTy, metaTy };
|
|
return llvm::FunctionType::get(voidTy, args, false);
|
|
}
|
|
|
|
case ValueWitness::Size:
|
|
case ValueWitness::Stride:
|
|
return IGM.SizeTy;
|
|
case ValueWitness::Flags:
|
|
case ValueWitness::ExtraInhabitantCount:
|
|
return IGM.Int32Ty;
|
|
}
|
|
|
|
llvm_unreachable("bad value witness!");
|
|
}
|
|
|
|
static llvm::AttributeList getValueWitnessAttrs(IRGenModule &IGM,
|
|
ValueWitness index) {
|
|
assert(isValueWitnessFunction(index));
|
|
|
|
auto &ctx = IGM.getLLVMContext();
|
|
|
|
// All value witnesses are nounwind.
|
|
auto attrs =
|
|
llvm::AttributeList().addFnAttribute(ctx, llvm::Attribute::NoUnwind);
|
|
|
|
switch (index) {
|
|
// These have two arguments, but they can alias.
|
|
case ValueWitness::AssignWithCopy:
|
|
return attrs;
|
|
|
|
// These have one argument.
|
|
case ValueWitness::Destroy:
|
|
case ValueWitness::DestructiveInjectEnumTag:
|
|
case ValueWitness::DestructiveProjectEnumData:
|
|
case ValueWitness::GetEnumTag:
|
|
case ValueWitness::StoreEnumTagSinglePayload:
|
|
return attrs.addParamAttribute(ctx, 0, llvm::Attribute::NoAlias);
|
|
|
|
case ValueWitness::GetEnumTagSinglePayload:
|
|
return attrs
|
|
.addFnAttribute(ctx, llvm::Attribute::getWithMemoryEffects(
|
|
ctx, llvm::MemoryEffects::readOnly()))
|
|
.addParamAttribute(ctx, 0, llvm::Attribute::NoAlias);
|
|
|
|
// These have two arguments and they don't alias each other.
|
|
case ValueWitness::AssignWithTake:
|
|
case ValueWitness::InitializeBufferWithCopyOfBuffer:
|
|
case ValueWitness::InitializeWithCopy:
|
|
case ValueWitness::InitializeWithTake:
|
|
return attrs.addParamAttribute(ctx, 0, llvm::Attribute::NoAlias)
|
|
.addParamAttribute(ctx, 1, llvm::Attribute::NoAlias);
|
|
|
|
case ValueWitness::Size:
|
|
case ValueWitness::Flags:
|
|
case ValueWitness::ExtraInhabitantCount:
|
|
case ValueWitness::Stride:
|
|
llvm_unreachable("not a function value witness");
|
|
}
|
|
llvm_unreachable("bad witness");
|
|
}
|
|
|
|
/// Return the cached pointer-to-function type for the given value
|
|
/// witness index.
|
|
llvm::Type *IRGenModule::getValueWitnessTy(ValueWitness index) {
|
|
assert(unsigned(index) < MaxNumValueWitnesses);
|
|
auto &ty = ValueWitnessTys[unsigned(index)];
|
|
if (ty) return ty;
|
|
|
|
ty = createWitnessType(*this, index);
|
|
return ty;
|
|
}
|
|
|
|
Signature IRGenModule::getValueWitnessSignature(ValueWitness index) {
|
|
assert(isValueWitnessFunction(index));
|
|
auto fnTy = cast<llvm::FunctionType>(getValueWitnessTy(index));
|
|
auto attrs = getValueWitnessAttrs(*this, index);
|
|
return Signature(fnTy, attrs, DefaultCC);
|
|
}
|
|
|
|
static StringRef getValueWitnessLabel(ValueWitness index) {
|
|
switch (index) {
|
|
case ValueWitness::Destroy:
|
|
return "destroy";
|
|
case ValueWitness::InitializeBufferWithCopyOfBuffer:
|
|
return "initializeBufferWithCopyOfBuffer";
|
|
case ValueWitness::AssignWithCopy:
|
|
return "assignWithCopy";
|
|
case ValueWitness::AssignWithTake:
|
|
return "assignWithTake";
|
|
case ValueWitness::InitializeWithCopy:
|
|
return "initializeWithCopy";
|
|
case ValueWitness::InitializeWithTake:
|
|
return "initializeWithTake";
|
|
case ValueWitness::Size:
|
|
return "size";
|
|
case ValueWitness::Flags:
|
|
return "flags";
|
|
case ValueWitness::ExtraInhabitantCount:
|
|
return "extraInhabitantCount";
|
|
case ValueWitness::Stride:
|
|
return "stride";
|
|
case ValueWitness::GetEnumTag:
|
|
return "getEnumTag";
|
|
case ValueWitness::DestructiveProjectEnumData:
|
|
return "destructiveProjectEnumData";
|
|
case ValueWitness::DestructiveInjectEnumTag:
|
|
return "destructiveInjectEnumTag";
|
|
case ValueWitness::GetEnumTagSinglePayload:
|
|
return "getEnumTagSinglePayload";
|
|
case ValueWitness::StoreEnumTagSinglePayload:
|
|
return "storeEnumTagSinglePayload";
|
|
}
|
|
llvm_unreachable("bad value witness index");
|
|
}
|
|
|
|
static llvm::StructType *
|
|
getOrCreateValueWitnessTableTy(IRGenModule &IGM, llvm::StructType *&cache,
|
|
StringRef name, bool includeEnumWitnesses) {
|
|
if (cache) return cache;
|
|
|
|
SmallVector<llvm::Type*, 16> types;
|
|
|
|
#define FUNC(lowerId, upperId, retTy, paramTys) \
|
|
types.push_back(IGM.Int8PtrTy);
|
|
#define DATA(lowerId, upperId, ty) \
|
|
types.push_back(IGM.getValueWitnessTy(ValueWitness::upperId));
|
|
|
|
// Add the base value witnesses.
|
|
#define WANT_ONLY_REQUIRED_VALUE_WITNESSES
|
|
#define FUNCTION_VALUE_WITNESS FUNC
|
|
#define DATA_VALUE_WITNESS DATA
|
|
#include "swift/ABI/ValueWitness.def"
|
|
|
|
// Add the enum value witnesses.
|
|
if (includeEnumWitnesses) {
|
|
#define WANT_ONLY_ENUM_VALUE_WITNESSES
|
|
#define FUNCTION_VALUE_WITNESS FUNC
|
|
#define DATA_VALUE_WITNESS DATA
|
|
#include "swift/ABI/ValueWitness.def"
|
|
}
|
|
|
|
#undef DATA
|
|
#undef FUNC
|
|
|
|
auto structTy = llvm::StructType::create(types, name);
|
|
cache = structTy;
|
|
return structTy;
|
|
}
|
|
|
|
llvm::PointerType *
|
|
IRGenModule::getOpaquePointerType(unsigned AddressSpace) const {
|
|
return llvm::PointerType::get(getLLVMContext(), AddressSpace);
|
|
}
|
|
|
|
llvm::StructType *IRGenModule::getValueWitnessTableTy() {
|
|
return getOrCreateValueWitnessTableTy(*this, ValueWitnessTableTy,
|
|
"swift.vwtable", false);
|
|
}
|
|
|
|
llvm::StructType *IRGenModule::getEnumValueWitnessTableTy() {
|
|
return getOrCreateValueWitnessTableTy(*this, EnumValueWitnessTableTy,
|
|
"swift.enum_vwtable", true);
|
|
}
|
|
|
|
Address irgen::slotForLoadOfOpaqueWitness(IRGenFunction &IGF,
|
|
llvm::Value *table,
|
|
ComputedWitnessIndex index,
|
|
bool areEntriesRelative) {
|
|
assert(table->getType() == IGF.IGM.WitnessTablePtrTy);
|
|
|
|
llvm::Value *slot = table;
|
|
llvm::Type *elementTy = table->getType();
|
|
Alignment align = IGF.IGM.getPointerAlignment();
|
|
|
|
// Are we loading from a relative protocol witness table?
|
|
if (areEntriesRelative) {
|
|
slot =
|
|
IGF.Builder.CreateBitOrPointerCast(table, IGF.IGM.RelativeAddressPtrTy);
|
|
elementTy = IGF.IGM.RelativeAddressTy;
|
|
align = Alignment(4);
|
|
}
|
|
|
|
// Is the index statically known?
|
|
if (auto staticIdx = index.getStaticIndex()) {
|
|
// GEP to the appropriate index, avoiding spurious IR in the trivial case.
|
|
if (staticIdx->getValue() != 0)
|
|
slot = IGF.Builder.CreateConstInBoundsGEP1_32(elementTy, slot,
|
|
staticIdx->getValue());
|
|
|
|
return Address(slot, elementTy, align);
|
|
}
|
|
|
|
// For a non-static index, we can't avoid a GEP.
|
|
slot =
|
|
IGF.Builder.CreateInBoundsGEP(elementTy, slot, index.getDynamicIndex());
|
|
return Address(slot, elementTy, align);
|
|
}
|
|
|
|
/// Load a specific witness from a known table. The result is
|
|
/// always an i8*.
|
|
llvm::Value *irgen::emitInvariantLoadOfOpaqueWitness(IRGenFunction &IGF,
|
|
bool isProtocolWitness,
|
|
llvm::Value *table,
|
|
ComputedWitnessIndex index,
|
|
llvm::Value **slotPtr) {
|
|
// Is this is a load of a relative protocol witness table entry.
|
|
auto isRelativeTable = IGF.IGM.IRGen.Opts.UseRelativeProtocolWitnessTables &&
|
|
isProtocolWitness;
|
|
|
|
auto slot = slotForLoadOfOpaqueWitness(IGF, table, index, isRelativeTable);
|
|
if (slotPtr) *slotPtr = slot.getAddress();
|
|
|
|
if (isRelativeTable) {
|
|
return IGF.emitLoadOfRelativePointer(slot, false, IGF.IGM.Int8Ty);
|
|
}
|
|
|
|
return IGF.emitInvariantLoad(slot);
|
|
}
|
|
|
|
/// Load a specific witness from a known table. The result is
|
|
/// always an i8*.
|
|
llvm::Value *irgen::emitInvariantLoadOfOpaqueWitness(IRGenFunction &IGF,
|
|
bool isProtocolWitness,
|
|
llvm::Value *table,
|
|
llvm::Value *index,
|
|
llvm::Value **slotPtr) {
|
|
assert(table->getType() == IGF.IGM.WitnessTablePtrTy);
|
|
assert(!isProtocolWitness &&
|
|
"This function does not yet support relative protocol witnesses");
|
|
|
|
// GEP to the appropriate index.
|
|
llvm::Value *slot =
|
|
IGF.Builder.CreateInBoundsGEP(IGF.IGM.WitnessTableTy, table, index);
|
|
|
|
if (slotPtr) *slotPtr = slot;
|
|
|
|
auto witness = IGF.Builder.CreateLoad(
|
|
Address(slot, IGF.IGM.WitnessTableTy, IGF.IGM.getPointerAlignment()));
|
|
IGF.setInvariantLoad(witness);
|
|
return witness;
|
|
}
|
|
|
|
static Address emitAddressOfValueWitnessTableValue(IRGenFunction &IGF,
|
|
llvm::Value *table,
|
|
ValueWitness witness) {
|
|
assert(!isValueWitnessFunction(witness));
|
|
assert(unsigned(witness) <= unsigned(ValueWitness::ExtraInhabitantCount) &&
|
|
"extraInhabitantCount not the last non-function value witness");
|
|
|
|
auto pointerSize = IGF.IGM.getPointerSize();
|
|
|
|
// Most of the witnesses are at an offset that's just a multiple of the
|
|
// pointer size, but the extra-inhabitant count is packed in after the
|
|
// 32-bit flags.
|
|
// This computation is correct for all pointer sizes, including 16.
|
|
// It would be wrong if size_t is ever a different size from the pointer
|
|
// size, though.
|
|
Size offset =
|
|
(witness == ValueWitness::ExtraInhabitantCount
|
|
? unsigned(ValueWitness::Flags) * pointerSize + Size(4)
|
|
: unsigned(witness) * pointerSize);
|
|
|
|
Address addr =
|
|
Address(table, IGF.IGM.WitnessTableTy, IGF.IGM.getPointerAlignment());
|
|
addr =
|
|
IGF.Builder.CreateElementBitCast(addr, IGF.IGM.getValueWitnessTableTy());
|
|
addr = IGF.Builder.CreateStructGEP(addr, unsigned(witness), offset);
|
|
return addr;
|
|
}
|
|
|
|
/// Given a value witness table, load one of the value witnesses.
|
|
/// The result has the appropriate type for the witness.
|
|
static llvm::Value *emitLoadOfValueWitnessValue(IRGenFunction &IGF,
|
|
llvm::Value *table,
|
|
ValueWitness witness) {
|
|
auto addr = emitAddressOfValueWitnessTableValue(IGF, table, witness);
|
|
auto load = IGF.Builder.CreateLoad(addr, getValueWitnessLabel(witness));
|
|
IGF.setInvariantLoad(load);
|
|
return load;
|
|
}
|
|
|
|
/// Given a type metadata pointer, load one of the value witnesses from its
|
|
/// value witness table.
|
|
static llvm::Value *
|
|
emitLoadOfValueWitnessValueFromMetadata(IRGenFunction &IGF,
|
|
llvm::Value *metadata,
|
|
ValueWitness index) {
|
|
llvm::Value *vwtable = IGF.emitValueWitnessTableRefForMetadata(metadata);
|
|
return emitLoadOfValueWitnessValue(IGF, vwtable, index);
|
|
}
|
|
|
|
/// Given a value witness table, load one of the value witnesses.
|
|
/// The result has the appropriate type for the witness.
|
|
static FunctionPointer emitLoadOfValueWitnessFunction(IRGenFunction &IGF,
|
|
llvm::Value *table,
|
|
ValueWitness index) {
|
|
assert(isValueWitnessFunction(index));
|
|
WitnessIndex windex = [&] {
|
|
unsigned i = unsigned(index);
|
|
if (i > unsigned(ValueWitness::Flags)) {
|
|
if (IGF.IGM.getPointerSize() == Size(8)) {
|
|
--i; // one pointer width skips both flags and xiCount
|
|
} else if (IGF.IGM.getPointerSize() == Size(4)) {
|
|
// no adjustment required
|
|
} else {
|
|
assert(IGF.IGM.getPointerSize() == Size(2));
|
|
i += 2; // flags and xiCount take up two pointers apiece
|
|
}
|
|
}
|
|
return WitnessIndex(i, false);
|
|
}();
|
|
|
|
llvm::Value *slot;
|
|
llvm::Value *witness =
|
|
emitInvariantLoadOfOpaqueWitness(IGF, /*isProtocolWitness*/false, table,
|
|
windex, &slot);
|
|
auto label = getValueWitnessLabel(index);
|
|
auto signature = IGF.IGM.getValueWitnessSignature(index);
|
|
|
|
witness = IGF.Builder.CreateBitCast(witness, IGF.IGM.PtrTy, label);
|
|
|
|
auto authInfo = PointerAuthInfo::emit(IGF,
|
|
IGF.getOptions().PointerAuth.ValueWitnesses,
|
|
slot, index);
|
|
|
|
witness->setName(getValueWitnessName(index));
|
|
return FunctionPointer::createSigned(FunctionPointer::Kind::Function, witness,
|
|
authInfo, signature);
|
|
}
|
|
|
|
/// Given a type metadata pointer, load one of the function
|
|
/// value witnesses from its value witness table.
|
|
static FunctionPointer
|
|
emitLoadOfValueWitnessFunctionFromMetadata(IRGenFunction &IGF,
|
|
llvm::Value *metadata,
|
|
ValueWitness index) {
|
|
llvm::Value *vwtable = IGF.emitValueWitnessTableRefForMetadata(metadata);
|
|
return emitLoadOfValueWitnessFunction(IGF, vwtable, index);
|
|
}
|
|
|
|
llvm::Value *IRGenFunction::emitValueWitnessValue(SILType type,
|
|
ValueWitness index) {
|
|
assert(!isValueWitnessFunction(index));
|
|
|
|
auto key = LocalTypeDataKind::forValueWitness(index);
|
|
if (auto witness = tryGetLocalTypeDataForLayout(type, key)) {
|
|
return witness;
|
|
}
|
|
|
|
auto vwtable = emitValueWitnessTableRef(type);
|
|
auto witness = emitLoadOfValueWitnessValue(*this, vwtable, index);
|
|
setScopedLocalTypeDataForLayout(type, key, witness);
|
|
return witness;
|
|
}
|
|
|
|
FunctionPointer
|
|
IRGenFunction::emitValueWitnessFunctionRef(SILType type,
|
|
llvm::Value *&metadataSlot,
|
|
ValueWitness index) {
|
|
assert(isValueWitnessFunction(index));
|
|
|
|
auto key = LocalTypeDataKind::forValueWitness(index);
|
|
if (auto witness = tryGetLocalTypeDataForLayout(type, key)) {
|
|
metadataSlot = emitTypeMetadataRefForLayout(type);
|
|
auto signature = IGM.getValueWitnessSignature(index);
|
|
PointerAuthInfo authInfo;
|
|
if (auto &schema = getOptions().PointerAuth.ValueWitnesses) {
|
|
auto discriminator =
|
|
tryGetLocalTypeDataForLayout(type,
|
|
LocalTypeDataKind::forValueWitnessDiscriminator(index));
|
|
assert(discriminator && "no saved discriminator for value witness fn!");
|
|
authInfo = PointerAuthInfo(schema.getKey(), discriminator);
|
|
}
|
|
return FunctionPointer::createSigned(FunctionPointer::Kind::Function,
|
|
witness, authInfo, signature);
|
|
}
|
|
|
|
auto vwtable = emitValueWitnessTableRef(type, &metadataSlot);
|
|
auto witness = emitLoadOfValueWitnessFunction(*this, vwtable, index);
|
|
setScopedLocalTypeDataForLayout(type, key, witness.getRawPointer());
|
|
if (auto &authInfo = witness.getAuthInfo()) {
|
|
setScopedLocalTypeDataForLayout(type,
|
|
LocalTypeDataKind::forValueWitnessDiscriminator(index),
|
|
authInfo.getDiscriminator());
|
|
}
|
|
|
|
return witness;
|
|
}
|
|
|
|
static llvm::Value *emitCastToOpaquePtr(IRGenFunction &IGF,
|
|
Address object) {
|
|
return IGF.Builder.CreateBitCast(object.getAddress(), IGF.IGM.OpaquePtrTy);
|
|
}
|
|
|
|
llvm::Value *irgen::emitInitializeBufferWithCopyOfBufferCall(IRGenFunction &IGF,
|
|
SILType T,
|
|
Address destBuffer,
|
|
Address srcBuffer) {
|
|
auto metadata = IGF.emitTypeMetadataRefForLayout(T);
|
|
return emitInitializeBufferWithCopyOfBufferCall(IGF, metadata,
|
|
destBuffer, srcBuffer);
|
|
}
|
|
|
|
/// Emit a call to do an 'initializeBufferWithCopyOfBuffer' operation.
|
|
llvm::Value *
|
|
irgen::emitInitializeBufferWithCopyOfBufferCall(IRGenFunction &IGF,
|
|
llvm::Value *metadata,
|
|
Address destBuffer,
|
|
Address srcBuffer) {
|
|
auto copyFn = emitLoadOfValueWitnessFunctionFromMetadata(IGF, metadata,
|
|
ValueWitness::InitializeBufferWithCopyOfBuffer);
|
|
llvm::CallInst *call =
|
|
IGF.Builder.CreateCall(copyFn,
|
|
{destBuffer.getAddress(), srcBuffer.getAddress(), metadata});
|
|
|
|
return call;
|
|
}
|
|
|
|
static llvm::Value *emitArraySizeInBytes(IRGenFunction &IGF,
|
|
Size eltSize,
|
|
llvm::Value *arrayLength) {
|
|
if (eltSize == Size(1)) {
|
|
return arrayLength;
|
|
} else {
|
|
return IGF.Builder.CreateMul(arrayLength, IGF.IGM.getSize(eltSize));
|
|
}
|
|
}
|
|
|
|
static llvm::Value *emitArraySizeInBytes(IRGenFunction &IGF,
|
|
llvm::Type *eltTy,
|
|
llvm::Value *arrayLength) {
|
|
auto eltSize = Size(IGF.IGM.DataLayout.getTypeAllocSize(eltTy));
|
|
return emitArraySizeInBytes(IGF, eltSize, arrayLength);
|
|
}
|
|
|
|
static Size getMaxVoluntaryStackAlloc(IRGenModule &IGM) {
|
|
return 16 * IGM.getPointerSize();
|
|
}
|
|
|
|
static bool isInEntryBlock(IRGenFunction &IGF) {
|
|
return IGF.Builder.GetInsertBlock() == &*IGF.CurFn->begin();
|
|
}
|
|
|
|
static bool isAcceptableStackAllocSize(IRGenFunction &IGF,
|
|
const llvm::APInt &size) {
|
|
return (isInEntryBlock(IGF) ||
|
|
size.ule(getMaxVoluntaryStackAlloc(IGF.IGM).getValue()));
|
|
}
|
|
|
|
static bool isAcceptableArrayStackAllocSize(IRGenFunction &IGF,
|
|
Size eltSize,
|
|
const llvm::APInt &arraySize) {
|
|
if (isInEntryBlock(IGF)) return true;
|
|
|
|
return (arraySize * eltSize.getValue()).ule(
|
|
getMaxVoluntaryStackAlloc(IGF.IGM).getValue());
|
|
}
|
|
|
|
StackAddress
|
|
IRGenFunction::emitArrayStackAllocation(llvm::Type *eltTy,
|
|
llvm::Value *arrayLength,
|
|
Alignment align,
|
|
StackAllocationIsNested_t isNested,
|
|
const llvm::Twine &name) {
|
|
Size eltSize = Size(IGM.DataLayout.getTypeAllocSize(eltTy));
|
|
|
|
// Allocate constant-sized allocations directly in the LLVM frame.
|
|
// We limit this unless we're emitting in the entry block.
|
|
if (auto constantLength = dyn_cast<llvm::ConstantInt>(arrayLength)) {
|
|
if (isAcceptableArrayStackAllocSize(*this, eltSize,
|
|
constantLength->getValue())) {
|
|
return emitStaticArrayAlloca(eltTy, eltSize, constantLength, align, name);
|
|
}
|
|
}
|
|
|
|
// If this code is properly-nested, use a dynamic allocation path.
|
|
if (isNested) {
|
|
return emitDynamicAlloca(eltTy, arrayLength, align, AllowsTaskAlloc,
|
|
/*mallocTypeId=*/nullptr, name);
|
|
}
|
|
|
|
// Otherwise, use the non-nested dynamic allocation method (generally
|
|
// malloc).
|
|
auto byteCount = emitArraySizeInBytes(*this, eltSize, arrayLength);
|
|
auto allocation = emitNonNestedStackAllocation(byteCount, align, name);
|
|
auto castedAddress =
|
|
Builder.CreateElementBitCast(allocation.getAddress(), eltTy);
|
|
return allocation.withAddress(castedAddress);
|
|
}
|
|
|
|
StackAddress
|
|
IRGenFunction::emitStackAllocation(llvm::Value *size, Alignment align,
|
|
StackAllocationIsNested_t isNested,
|
|
const llvm::Twine &name) {
|
|
// Allocate constant-sized allocations directly in the LLVM frame.
|
|
// We limit this unless we're emitting in the entry block.
|
|
if (auto constantSize = dyn_cast<llvm::ConstantInt>(size)) {
|
|
if (isAcceptableStackAllocSize(*this, constantSize->getValue())) {
|
|
return emitStaticByteArrayAlloca(constantSize, align, name);
|
|
}
|
|
}
|
|
|
|
// If this code is properly-nested, use a dynamic allocation path.
|
|
if (isNested) {
|
|
return emitDynamicAlloca(IGM.Int8Ty, size, align, AllowsTaskAlloc,
|
|
/*mallocTypeId=*/nullptr, name);
|
|
}
|
|
|
|
// Otherwise, use the non-nested dynamic allocation method (generally
|
|
// malloc).
|
|
return emitNonNestedStackAllocation(size, align, name);
|
|
}
|
|
|
|
void IRGenFunction::emitStackDeallocation(StackAddress address) {
|
|
switch (address.getKind()) {
|
|
case StackAddress::StaticAlloca:
|
|
emitDeallocateStaticAlloca(address);
|
|
return;
|
|
|
|
case StackAddress::DynamicAlloca:
|
|
case StackAddress::TaskAlloc:
|
|
case StackAddress::CoroAlloc:
|
|
emitDeallocateDynamicAlloca(address);
|
|
return;
|
|
|
|
case StackAddress::NonNested:
|
|
emitNonNestedStackDeallocation(address);
|
|
return;
|
|
}
|
|
llvm_unreachable("bad stack address kind");
|
|
}
|
|
|
|
StackAddress
|
|
IRGenFunction::emitStaticAlloca(llvm::Type *ty, Size size,
|
|
Alignment align, const llvm::Twine &name) {
|
|
auto addr = createAlloca(ty, align, name);
|
|
|
|
// LLVM IR requires the size to be an i64.
|
|
auto sizeForLifetime = llvm::ConstantInt::get(IGM.Int64Ty, size.getValue());
|
|
Builder.CreateLifetimeStart(addr, sizeForLifetime);
|
|
|
|
return StackAddress(addr, StackAddress::StaticAlloca, sizeForLifetime);
|
|
}
|
|
|
|
StackAddress
|
|
IRGenFunction::emitStaticByteArrayAlloca(llvm::ConstantInt *size,
|
|
Alignment align,
|
|
const llvm::Twine &name) {
|
|
return emitStaticArrayAlloca(IGM.Int8Ty, Size(1), size, align, name);
|
|
}
|
|
|
|
StackAddress
|
|
IRGenFunction::emitStaticArrayAlloca(llvm::Type *eltTy, Size eltSize,
|
|
llvm::ConstantInt *arrayLength,
|
|
Alignment align,
|
|
const llvm::Twine &name) {
|
|
auto addr = createAlloca(eltTy, arrayLength, align, name);
|
|
|
|
// The lifetime intrinsics require an i64 on all targets.
|
|
auto sizeForLifetime = arrayLength;
|
|
if (eltSize != Size(1) || arrayLength->getType() != IGM.Int64Ty) {
|
|
sizeForLifetime = llvm::ConstantInt::get(IGM.Int64Ty,
|
|
eltSize.getValue() * arrayLength->getZExtValue());
|
|
}
|
|
|
|
Builder.CreateLifetimeStart(addr, sizeForLifetime);
|
|
|
|
return StackAddress(addr, StackAddress::StaticAlloca, sizeForLifetime);
|
|
}
|
|
|
|
void
|
|
IRGenFunction::emitDeallocateStaticAlloca(StackAddress address) {
|
|
assert(address.isValid());
|
|
assert(address.getKind() == StackAddress::StaticAlloca);
|
|
|
|
auto csize = address.getExtraInfo();
|
|
assert(csize || isa<llvm::UndefValue>(address.getAddressPointer()));
|
|
if (!csize) return;
|
|
|
|
Builder.CreateLifetimeEnd(address.getAddress(),
|
|
cast<llvm::ConstantInt>(csize));
|
|
}
|
|
|
|
StackAddress IRGenFunction::emitDynamicStackAllocation(
|
|
SILType T, StackAllocationIsNested_t isNested, const llvm::Twine &name) {
|
|
if (isNested) {
|
|
return emitDynamicAlloca(T, name);
|
|
}
|
|
|
|
// TODO: Alignment should be platform specific.
|
|
auto *size = emitLoadOfSize(*this, T);
|
|
auto align = Alignment(MaximumAlignment);
|
|
return emitNonNestedStackAllocation(size, align, name);
|
|
}
|
|
|
|
void IRGenFunction::emitDynamicStackDeallocation(StackAddress address) {
|
|
assert(address.getKind() != StackAddress::StaticAlloca);
|
|
if (address.getKind() != StackAddress::NonNested) {
|
|
return emitDeallocateDynamicAlloca(address);
|
|
}
|
|
|
|
return emitNonNestedStackDeallocation(address);
|
|
}
|
|
|
|
StackAddress IRGenFunction::emitNonNestedStackAllocation(
|
|
llvm::Value *size, Alignment align, const llvm::Twine &name) {
|
|
// First malloc the memory.
|
|
auto mallocFn = IGM.getMallocFunctionPointer();
|
|
auto *call = Builder.CreateCall(mallocFn, {size});
|
|
call->setDoesNotThrow();
|
|
call->setCallingConv(IGM.C_CC);
|
|
|
|
auto address = Address(call, IGM.Int8Ty, align);
|
|
return StackAddress(address, StackAddress::NonNested, call);
|
|
}
|
|
|
|
void IRGenFunction::emitNonNestedStackDeallocation(StackAddress address) {
|
|
assert(address.getKind() == StackAddress::NonNested);
|
|
auto *call = Builder.CreateCall(IGM.getFreeFunctionPointer(),
|
|
{address.getExtraInfo()});
|
|
call->setDoesNotThrow();
|
|
call->setCallingConv(IGM.C_CC);
|
|
}
|
|
|
|
/// Emit a dynamic alloca call to allocate enough memory to hold an object of
|
|
/// type 'T' and an optional llvm.stackrestore point if 'isInEntryBlock' is
|
|
/// false.
|
|
StackAddress IRGenFunction::emitDynamicAlloca(SILType T,
|
|
const llvm::Twine &name) {
|
|
llvm::Value *size = emitLoadOfSize(*this, T);
|
|
return emitDynamicAlloca(IGM.Int8Ty, size, Alignment(MaximumAlignment),
|
|
AllowsTaskAlloc,
|
|
/*mallocTypeId=*/nullptr, name);
|
|
}
|
|
|
|
StackAddress IRGenFunction::emitDynamicAlloca(llvm::Type *eltTy,
|
|
llvm::Value *arraySize,
|
|
Alignment align,
|
|
AllowsTaskAlloc_t allowTaskAlloc,
|
|
llvm::Value *mallocTypeId,
|
|
const llvm::Twine &name) {
|
|
// Async functions call task alloc.
|
|
if (allowTaskAlloc && isAsync()) {
|
|
llvm::Value *byteCount = emitArraySizeInBytes(*this, eltTy, arraySize);
|
|
// The task allocator wants size increments in the multiple of
|
|
// MaximumAlignment.
|
|
byteCount = alignUpToMaximumAlignment(IGM.SizeTy, byteCount);
|
|
auto allocation = emitTaskAlloc(byteCount, align);
|
|
auto address = Builder.CreateElementBitCast(allocation, eltTy);
|
|
return StackAddress(address, StackAddress::TaskAlloc,
|
|
allocation.getAddress());
|
|
|
|
// In coroutines, call llvm.coro.alloca.alloc.
|
|
} else if (isCoroutine()) {
|
|
// NOTE: llvm does not support dynamic allocas in coroutines.
|
|
|
|
// Compute the number of bytes to allocate.
|
|
llvm::Value *byteCount = emitArraySizeInBytes(*this, eltTy, arraySize);
|
|
|
|
auto alignment = llvm::ConstantInt::get(IGM.Int32Ty, align.getValue());
|
|
|
|
// Allocate memory. This produces an abstract token.
|
|
llvm::SmallVector<llvm::Value *, 4> args = {byteCount, alignment};
|
|
if (mallocTypeId) {
|
|
args.push_back(mallocTypeId);
|
|
}
|
|
auto *allocToken = Builder.CreateIntrinsicCall(
|
|
mallocTypeId ? llvm::Intrinsic::coro_alloca_alloc_frame
|
|
: llvm::Intrinsic::coro_alloca_alloc,
|
|
{IGM.SizeTy}, args);
|
|
|
|
// Get the allocation result.
|
|
auto ptr = Builder.CreateIntrinsicCall(llvm::Intrinsic::coro_alloca_get,
|
|
{allocToken});
|
|
|
|
auto address =
|
|
Builder.CreateElementBitCast(Address(ptr, IGM.Int8Ty, align), eltTy);
|
|
return StackAddress(address, StackAddress::CoroAlloc, allocToken);
|
|
}
|
|
|
|
// Otherwise, use a dynamic alloca.
|
|
llvm::Value *stackRestorePoint = nullptr;
|
|
|
|
// Save the stack pointer if we are not in the entry block (we could be
|
|
// executed more than once).
|
|
bool isInEntryBlock = ::isInEntryBlock(*this);
|
|
if (!isInEntryBlock) {
|
|
stackRestorePoint = Builder.CreateIntrinsicCall(
|
|
llvm::Intrinsic::stacksave,
|
|
{IGM.DataLayout.getAllocaPtrType(IGM.getLLVMContext())}, {}, "spsave");
|
|
}
|
|
|
|
// Emit the dynamic alloca. We delete CreateAlloca on our IRBuilder
|
|
// subclass because we don't want code to naively create dynamic allocas,
|
|
// but in this case we really do want to use it, so we have to look
|
|
// around that.
|
|
auto *alloca = Builder.IRBuilderBase::CreateAlloca(eltTy, arraySize, name);
|
|
alloca->setAlignment(llvm::MaybeAlign(align.getValue()).valueOrOne());
|
|
|
|
assert(!isInEntryBlock ||
|
|
getActiveDominancePoint().isUniversal() &&
|
|
"Must be in entry block if we insert dynamic alloca's without "
|
|
"stackrestores");
|
|
auto addr = Address(alloca, eltTy, align);
|
|
return StackAddress(addr, StackAddress::DynamicAlloca, stackRestorePoint);
|
|
}
|
|
|
|
/// Deallocate dynamic alloca's memory if requested by restoring the stack
|
|
/// location before the dynamic alloca's call.
|
|
void IRGenFunction::emitDeallocateDynamicAlloca(StackAddress address,
|
|
bool useTaskDeallocThrough,
|
|
bool forCalleeCoroutineFrame) {
|
|
assert(address.getKind() == StackAddress::DynamicAlloca ||
|
|
address.getKind() == StackAddress::TaskAlloc ||
|
|
address.getKind() == StackAddress::CoroAlloc);
|
|
|
|
if (!address.getAddress().isValid())
|
|
return;
|
|
|
|
// Async function use taskDealloc.
|
|
if (address.getKind() == StackAddress::TaskAlloc) {
|
|
assert(isAsync());
|
|
if (useTaskDeallocThrough) {
|
|
emitTaskDeallocThrough(
|
|
Address(address.getExtraInfo(), IGM.Int8Ty, address.getAlignment()));
|
|
return;
|
|
}
|
|
emitTaskDealloc(
|
|
Address(address.getExtraInfo(), IGM.Int8Ty, address.getAlignment()));
|
|
return;
|
|
}
|
|
|
|
// In coroutines, unconditionally call llvm.coro.alloca.free.
|
|
// Except if the address is invalid, this happens when this is a StackAddress
|
|
// for a partial_apply [stack] that did not need a context object on the
|
|
// stack.
|
|
if (address.getKind() == StackAddress::CoroAlloc) {
|
|
assert(isCoroutine());
|
|
|
|
auto allocToken = address.getExtraInfo();
|
|
if (!allocToken) {
|
|
#ifndef NDEBUG
|
|
auto *alloca = cast<llvm::AllocaInst>(address.getAddress().getAddress());
|
|
assert(isa<llvm::ConstantInt>(alloca->getArraySize()) &&
|
|
"Dynamic alloca without a token?!");
|
|
#endif
|
|
return;
|
|
}
|
|
Builder.CreateIntrinsicCall(forCalleeCoroutineFrame
|
|
? llvm::Intrinsic::coro_alloca_free_frame
|
|
: llvm::Intrinsic::coro_alloca_free,
|
|
allocToken);
|
|
return;
|
|
}
|
|
|
|
// Otherwise, call llvm.stackrestore if we performed a stacksave.
|
|
assert(address.getKind() == StackAddress::DynamicAlloca);
|
|
auto savedSP = address.getExtraInfo();
|
|
if (savedSP == nullptr)
|
|
return;
|
|
Builder.CreateIntrinsicCall(llvm::Intrinsic::stackrestore,
|
|
{savedSP->getType()}, {savedSP});
|
|
}
|
|
|
|
/// Emit a call to do an 'initializeArrayWithCopy' operation.
|
|
void irgen::emitInitializeArrayWithCopyCall(IRGenFunction &IGF,
|
|
SILType T,
|
|
Address destObject,
|
|
Address srcObject,
|
|
llvm::Value *count) {
|
|
auto metadata = IGF.emitTypeMetadataRefForLayout(T);
|
|
auto dest = emitCastToOpaquePtr(IGF, destObject);
|
|
auto src = emitCastToOpaquePtr(IGF, srcObject);
|
|
IGF.Builder.CreateCall(IGF.IGM.getArrayInitWithCopyFunctionPointer(),
|
|
{dest, src, count, metadata});
|
|
}
|
|
|
|
/// Emit a call to do an 'initializeArrayWithTakeNoAlias' operation.
|
|
void irgen::emitInitializeArrayWithTakeNoAliasCall(IRGenFunction &IGF,
|
|
SILType T,
|
|
Address destObject,
|
|
Address srcObject,
|
|
llvm::Value *count) {
|
|
auto metadata = IGF.emitTypeMetadataRefForLayout(T);
|
|
auto dest = emitCastToOpaquePtr(IGF, destObject);
|
|
auto src = emitCastToOpaquePtr(IGF, srcObject);
|
|
IGF.Builder.CreateCall(IGF.IGM.getArrayInitWithTakeNoAliasFunctionPointer(),
|
|
{dest, src, count, metadata});
|
|
}
|
|
|
|
/// Emit a call to do an 'initializeArrayWithTakeFrontToBack' operation.
|
|
void irgen::emitInitializeArrayWithTakeFrontToBackCall(IRGenFunction &IGF,
|
|
SILType T,
|
|
Address destObject,
|
|
Address srcObject,
|
|
llvm::Value *count) {
|
|
auto metadata = IGF.emitTypeMetadataRefForLayout(T);
|
|
auto dest = emitCastToOpaquePtr(IGF, destObject);
|
|
auto src = emitCastToOpaquePtr(IGF, srcObject);
|
|
IGF.Builder.CreateCall(
|
|
IGF.IGM.getArrayInitWithTakeFrontToBackFunctionPointer(),
|
|
{dest, src, count, metadata});
|
|
}
|
|
|
|
/// Emit a call to do an 'initializeArrayWithTakeBackToFront' operation.
|
|
void irgen::emitInitializeArrayWithTakeBackToFrontCall(IRGenFunction &IGF,
|
|
SILType T,
|
|
Address destObject,
|
|
Address srcObject,
|
|
llvm::Value *count) {
|
|
auto metadata = IGF.emitTypeMetadataRefForLayout(T);
|
|
auto dest = emitCastToOpaquePtr(IGF, destObject);
|
|
auto src = emitCastToOpaquePtr(IGF, srcObject);
|
|
IGF.Builder.CreateCall(
|
|
IGF.IGM.getArrayInitWithTakeBackToFrontFunctionPointer(),
|
|
{dest, src, count, metadata});
|
|
}
|
|
|
|
/// Emit a call to do an 'assignWithCopy' operation.
|
|
void irgen::emitAssignWithCopyCall(IRGenFunction &IGF,
|
|
SILType T,
|
|
Address destObject,
|
|
Address srcObject) {
|
|
llvm::Value *metadata;
|
|
auto copyFn = IGF.emitValueWitnessFunctionRef(T, metadata,
|
|
ValueWitness::AssignWithCopy);
|
|
auto dest = emitCastToOpaquePtr(IGF, destObject);
|
|
auto src = emitCastToOpaquePtr(IGF, srcObject);
|
|
IGF.Builder.CreateCall(copyFn, {dest, src, metadata});
|
|
}
|
|
|
|
/// Emit a call to do an 'assignWithCopy' operation.
|
|
void irgen::emitAssignWithCopyCall(IRGenFunction &IGF,
|
|
llvm::Value *metadata,
|
|
Address destObject,
|
|
Address srcObject) {
|
|
auto copyFn = emitLoadOfValueWitnessFunctionFromMetadata(IGF, metadata,
|
|
ValueWitness::AssignWithCopy);
|
|
auto dest = emitCastToOpaquePtr(IGF, destObject);
|
|
auto src = emitCastToOpaquePtr(IGF, srcObject);
|
|
IGF.Builder.CreateCall(copyFn, {dest, src, metadata});
|
|
}
|
|
|
|
/// Emit a call to do an 'arrayAssignWithCopyNoAlias' operation.
|
|
void irgen::emitAssignArrayWithCopyNoAliasCall(IRGenFunction &IGF, SILType T,
|
|
Address destObject, Address srcObject,
|
|
llvm::Value *count) {
|
|
auto metadata = IGF.emitTypeMetadataRefForLayout(T);
|
|
auto dest = emitCastToOpaquePtr(IGF, destObject);
|
|
auto src = emitCastToOpaquePtr(IGF, srcObject);
|
|
IGF.Builder.CreateCall(IGF.IGM.getArrayAssignWithCopyNoAliasFunctionPointer(),
|
|
{dest, src, count, metadata});
|
|
}
|
|
|
|
/// Emit a call to do an 'arrayAssignWithCopyFrontToBack' operation.
|
|
void irgen::emitAssignArrayWithCopyFrontToBackCall(IRGenFunction &IGF,
|
|
SILType T,
|
|
Address destObject,
|
|
Address srcObject,
|
|
llvm::Value *count) {
|
|
auto metadata = IGF.emitTypeMetadataRefForLayout(T);
|
|
auto dest = emitCastToOpaquePtr(IGF, destObject);
|
|
auto src = emitCastToOpaquePtr(IGF, srcObject);
|
|
IGF.Builder.CreateCall(
|
|
IGF.IGM.getArrayAssignWithCopyFrontToBackFunctionPointer(),
|
|
{dest, src, count, metadata});
|
|
}
|
|
|
|
/// Emit a call to do an 'arrayAssignWithCopyBackToFront' operation.
|
|
void irgen::emitAssignArrayWithCopyBackToFrontCall(IRGenFunction &IGF,
|
|
SILType T,
|
|
Address destObject,
|
|
Address srcObject,
|
|
llvm::Value *count) {
|
|
auto metadata = IGF.emitTypeMetadataRefForLayout(T);
|
|
auto dest = emitCastToOpaquePtr(IGF, destObject);
|
|
auto src = emitCastToOpaquePtr(IGF, srcObject);
|
|
IGF.Builder.CreateCall(
|
|
IGF.IGM.getArrayAssignWithCopyBackToFrontFunctionPointer(),
|
|
{dest, src, count, metadata});
|
|
}
|
|
|
|
/// Emit a call to do an 'assignWithTake' operation.
|
|
void irgen::emitAssignWithTakeCall(IRGenFunction &IGF,
|
|
SILType T,
|
|
Address destObject,
|
|
Address srcObject) {
|
|
llvm::Value *metadata;
|
|
auto copyFn = IGF.emitValueWitnessFunctionRef(T, metadata,
|
|
ValueWitness::AssignWithTake);
|
|
auto dest = emitCastToOpaquePtr(IGF, destObject);
|
|
auto src = emitCastToOpaquePtr(IGF, srcObject);
|
|
IGF.Builder.CreateCall(copyFn, {dest, src, metadata});
|
|
}
|
|
|
|
/// Emit a call to do an 'arrayAssignWithTake' operation.
|
|
void irgen::emitAssignArrayWithTakeCall(IRGenFunction &IGF, SILType T,
|
|
Address destObject, Address srcObject,
|
|
llvm::Value *count) {
|
|
auto metadata = IGF.emitTypeMetadataRefForLayout(T);
|
|
auto dest = emitCastToOpaquePtr(IGF, destObject);
|
|
auto src = emitCastToOpaquePtr(IGF, srcObject);
|
|
IGF.Builder.CreateCall(IGF.IGM.getArrayAssignWithTakeFunctionPointer(),
|
|
{dest, src, count, metadata});
|
|
}
|
|
|
|
/// Emit a call to do a 'destroyArray' operation.
|
|
void irgen::emitDestroyArrayCall(IRGenFunction &IGF,
|
|
SILType T,
|
|
Address object,
|
|
llvm::Value *count) {
|
|
// If T is a trivial/POD type, nothing needs to be done.
|
|
if (IGF.IGM.getTypeProperties(T).isTrivial())
|
|
return;
|
|
|
|
auto metadata = IGF.emitTypeMetadataRefForLayout(T);
|
|
auto obj = emitCastToOpaquePtr(IGF, object);
|
|
IGF.Builder.CreateCall(IGF.IGM.getArrayDestroyFunctionPointer(),
|
|
{obj, count, metadata});
|
|
}
|
|
|
|
/// Emit a trampoline to call the getEnumTagSinglePayload witness. API:
|
|
/// UINT_TYPE (const T* enum, UINT_TYPE emptyCases, M *self)
|
|
static llvm::Constant *
|
|
getGetEnumTagSinglePayloadTrampolineFn(IRGenModule &IGM) {
|
|
|
|
llvm::Type *argTys[] = {IGM.OpaquePtrTy, IGM.Int32Ty, IGM.TypeMetadataPtrTy};
|
|
|
|
llvm::SmallString<40> fnName("__swift_getEnumTagSinglePayload");
|
|
|
|
auto func = IGM.getOrCreateHelperFunction(
|
|
fnName, IGM.Int32Ty, argTys,
|
|
[&](IRGenFunction &IGF) {
|
|
auto it = IGF.CurFn->arg_begin();
|
|
auto *enumAddr = &*(it++);
|
|
auto *numEmptyCases = &*(it++);
|
|
auto *metadata = &*(it++);
|
|
auto &Builder = IGF.Builder;
|
|
auto witnessFunc = emitLoadOfValueWitnessFunctionFromMetadata(
|
|
IGF, metadata, ValueWitness::GetEnumTagSinglePayload);
|
|
auto *result = Builder.CreateCall(witnessFunc,
|
|
{enumAddr, numEmptyCases, metadata});
|
|
Builder.CreateRet(result);
|
|
},
|
|
true /*noinline*/);
|
|
|
|
// This function is readonly.
|
|
cast<llvm::Function>(func)->setOnlyReadsMemory();
|
|
return func;
|
|
}
|
|
|
|
/// Emit a trampoline to call the storeEnumTagSinglePayload witness. API:
|
|
/// VOID_TYPE (const T* enum, UINT_TYPE whichCase, UINT_TYPE emptyCases,
|
|
/// M *self)
|
|
static llvm::Constant *
|
|
getStoreEnumTagSinglePayloadTrampolineFn(IRGenModule &IGM) {
|
|
|
|
llvm::Type *argTys[] = {IGM.OpaquePtrTy, IGM.Int32Ty, IGM.Int32Ty,
|
|
IGM.TypeMetadataPtrTy};
|
|
|
|
llvm::SmallString<40> fnName("__swift_storeEnumTagSinglePayload");
|
|
|
|
return IGM.getOrCreateHelperFunction(
|
|
fnName, IGM.VoidTy, argTys,
|
|
[&](IRGenFunction &IGF) {
|
|
auto it = IGF.CurFn->arg_begin();
|
|
auto *enumAddr = &*(it++);
|
|
auto *whichCase = &*(it++);
|
|
auto *numEmptyCases = &*(it++);
|
|
auto *metadata = &*(it++);
|
|
auto &Builder = IGF.Builder;
|
|
auto witnessFunc = emitLoadOfValueWitnessFunctionFromMetadata(
|
|
IGF, metadata, ValueWitness::StoreEnumTagSinglePayload);
|
|
Builder.CreateCall(witnessFunc,
|
|
{enumAddr, whichCase, numEmptyCases, metadata});
|
|
Builder.CreateRetVoid();
|
|
},
|
|
true /*noinline*/);
|
|
}
|
|
|
|
llvm::Value *irgen::emitGetEnumTagSinglePayloadCall(IRGenFunction &IGF,
|
|
SILType T,
|
|
llvm::Value *numEmptyCases,
|
|
Address destObject) {
|
|
if (!IGF.optimizeForSize()) {
|
|
llvm::Value *metadata;
|
|
auto fn = IGF.emitValueWitnessFunctionRef(
|
|
T, metadata, ValueWitness::GetEnumTagSinglePayload);
|
|
auto dest = emitCastToOpaquePtr(IGF, destObject);
|
|
llvm::CallInst *call = IGF.Builder.CreateCall(
|
|
fn, {dest, numEmptyCases, metadata});
|
|
return call;
|
|
}
|
|
auto *metadata = IGF.emitTypeMetadataRefForLayout(T);
|
|
auto *func = getGetEnumTagSinglePayloadTrampolineFn(IGF.IGM);
|
|
auto *fnType = cast<llvm::Function>(func)->getFunctionType();
|
|
auto dest = emitCastToOpaquePtr(IGF, destObject);
|
|
auto *result =
|
|
IGF.Builder.CreateCall(fnType, func, {dest, numEmptyCases, metadata});
|
|
return result;
|
|
}
|
|
|
|
void irgen::emitStoreEnumTagSinglePayloadCall(
|
|
IRGenFunction &IGF, SILType T, llvm::Value *whichCase,
|
|
llvm::Value *numEmptyCases, Address destObject) {
|
|
if (!IGF.optimizeForSize()) {
|
|
llvm::Value *metadata;
|
|
auto fn = IGF.emitValueWitnessFunctionRef(
|
|
T, metadata, ValueWitness::StoreEnumTagSinglePayload);
|
|
auto dest = emitCastToOpaquePtr(IGF, destObject);
|
|
IGF.Builder.CreateCall(fn, {dest, whichCase, numEmptyCases, metadata});
|
|
return;
|
|
}
|
|
|
|
auto *metadata = IGF.emitTypeMetadataRefForLayout(T);
|
|
auto *func = getStoreEnumTagSinglePayloadTrampolineFn(IGF.IGM);
|
|
auto *fnType = cast<llvm::Function>(func)->getFunctionType();
|
|
auto dest = emitCastToOpaquePtr(IGF, destObject);
|
|
IGF.Builder.CreateCall(fnType, func,
|
|
{dest, whichCase, numEmptyCases, metadata});
|
|
}
|
|
|
|
/// Emit a call to the 'getEnumTag' operation.
|
|
llvm::Value *irgen::emitGetEnumTagCall(IRGenFunction &IGF,
|
|
SILType T,
|
|
Address srcObject) {
|
|
llvm::Value *metadata;
|
|
auto fn = IGF.emitValueWitnessFunctionRef(T, metadata,
|
|
ValueWitness::GetEnumTag);
|
|
|
|
auto src = emitCastToOpaquePtr(IGF, srcObject);
|
|
llvm::CallInst *call =
|
|
IGF.Builder.CreateCall(fn, {src, metadata});
|
|
return call;
|
|
}
|
|
|
|
/// Emit a call to the 'destructiveProjectEnumData' operation.
|
|
/// The type must be dynamically known to have enum witnesses.
|
|
void irgen::emitDestructiveProjectEnumDataCall(IRGenFunction &IGF,
|
|
SILType T,
|
|
Address srcObject) {
|
|
llvm::Value *metadata;
|
|
auto fn = IGF.emitValueWitnessFunctionRef(T, metadata,
|
|
ValueWitness::DestructiveProjectEnumData);
|
|
auto src = emitCastToOpaquePtr(IGF, srcObject);
|
|
IGF.Builder.CreateCall(fn, {src, metadata});
|
|
}
|
|
|
|
/// Emit a call to the 'destructiveInjectEnumTag' operation.
|
|
/// The type must be dynamically known to have enum witnesses.
|
|
void irgen::emitDestructiveInjectEnumTagCall(IRGenFunction &IGF,
|
|
SILType T,
|
|
llvm::Value *tagValue,
|
|
Address srcObject) {
|
|
llvm::Value *metadata;
|
|
auto fn = IGF.emitValueWitnessFunctionRef(T, metadata,
|
|
ValueWitness::DestructiveInjectEnumTag);
|
|
auto src = emitCastToOpaquePtr(IGF, srcObject);
|
|
IGF.Builder.CreateCall(fn, {src, tagValue, metadata});
|
|
}
|
|
|
|
/// Load the 'size' value witness from the given table as a size_t.
|
|
llvm::Value *irgen::emitLoadOfSize(IRGenFunction &IGF, SILType T) {
|
|
return IGF.emitValueWitnessValue(T, ValueWitness::Size);
|
|
}
|
|
|
|
/// Load the 'alignmentMask' value witness from the given table as a size_t.
|
|
llvm::Value *irgen::emitLoadOfAlignmentMask(IRGenFunction &IGF, SILType T) {
|
|
auto flags = IGF.emitValueWitnessValue(T, ValueWitness::Flags);
|
|
return emitAlignMaskFromFlags(IGF, flags);
|
|
}
|
|
|
|
/// Load the 'isTriviallyDestroyable' valueWitness from the given table as an i1.
|
|
llvm::Value *irgen::emitLoadOfIsTriviallyDestroyable(IRGenFunction &IGF, SILType T) {
|
|
auto flags = IGF.emitValueWitnessValue(T, ValueWitness::Flags);
|
|
auto mask = IGF.IGM.getInt32(ValueWitnessFlags::IsNonPOD);
|
|
auto masked = IGF.Builder.CreateAnd(flags, mask);
|
|
return IGF.Builder.CreateICmpEQ(masked, IGF.IGM.getInt32(0),
|
|
flags->getName() + ".isTriviallyDestroyable");
|
|
}
|
|
|
|
/// Load the 'isBitwiseTakable' valueWitness from the given table as an i1.
|
|
llvm::Value *irgen::emitLoadOfIsBitwiseTakable(IRGenFunction &IGF, SILType T) {
|
|
auto flags = IGF.emitValueWitnessValue(T, ValueWitness::Flags);
|
|
auto mask = IGF.IGM.getInt32(ValueWitnessFlags::IsNonBitwiseTakable);
|
|
auto masked = IGF.Builder.CreateAnd(flags, mask);
|
|
return IGF.Builder.CreateICmpEQ(masked, IGF.IGM.getInt32(0),
|
|
flags->getName() + ".isBitwiseTakable");
|
|
}
|
|
|
|
/// Load the 'isBitwiseBorrowable' valueWitness from the given table as an i1.
|
|
llvm::Value *irgen::emitLoadOfIsBitwiseBorrowable(IRGenFunction &IGF, SILType T) {
|
|
auto flags = IGF.emitValueWitnessValue(T, ValueWitness::Flags);
|
|
|
|
// Bitwise-takable implies bitwise-borrowable, and the not-bitwise-borrowable
|
|
// bit may be left indeterminate if the type is already not-bitwise-takable.
|
|
// So the type is bitwise-borrowable only when both bits are zero.
|
|
auto mask = IGF.IGM.getInt32(ValueWitnessFlags::IsNonBitwiseBorrowable
|
|
| ValueWitnessFlags::IsNonBitwiseTakable);
|
|
auto masked = IGF.Builder.CreateAnd(flags, mask);
|
|
return IGF.Builder.CreateICmpEQ(masked, IGF.IGM.getInt32(0),
|
|
flags->getName() + ".isBitwiseBorrowable");
|
|
}
|
|
|
|
/// Load the 'isAddressableForDependencies' valueWitness from the given table as an i1.
|
|
llvm::Value *irgen::emitLoadOfIsAddressableForDependencies(IRGenFunction &IGF, SILType T) {
|
|
auto flags = IGF.emitValueWitnessValue(T, ValueWitness::Flags);
|
|
auto mask = IGF.IGM.getInt32(ValueWitnessFlags::IsAddressableForDependencies);
|
|
auto masked = IGF.Builder.CreateAnd(flags, mask);
|
|
return IGF.Builder.CreateICmpNE(masked, IGF.IGM.getInt32(0),
|
|
flags->getName() + ".isAddressableForDependencies");
|
|
}
|
|
|
|
/// Load the 'isInline' valueWitness from the given table as an i1.
|
|
llvm::Value *irgen::emitLoadOfIsInline(IRGenFunction &IGF, SILType T) {
|
|
auto flags = IGF.emitValueWitnessValue(T, ValueWitness::Flags);
|
|
auto mask = IGF.IGM.getInt32(ValueWitnessFlags::IsNonInline);
|
|
auto masked = IGF.Builder.CreateAnd(flags, mask);
|
|
return IGF.Builder.CreateICmpEQ(masked, IGF.IGM.getInt32(0),
|
|
flags->getName() + ".isInline");
|
|
}
|
|
|
|
/// Load the 'stride' value witness from the given table as a size_t.
|
|
llvm::Value *irgen::emitLoadOfStride(IRGenFunction &IGF, SILType T) {
|
|
return IGF.emitValueWitnessValue(T, ValueWitness::Stride);
|
|
}
|
|
|
|
llvm::Value *irgen::emitLoadOfExtraInhabitantCount(IRGenFunction &IGF,
|
|
SILType T) {
|
|
return IGF.emitValueWitnessValue(T, ValueWitness::ExtraInhabitantCount);
|
|
}
|
|
|
|
void irgen::emitStoreOfExtraInhabitantCount(IRGenFunction &IGF,
|
|
llvm::Value *value,
|
|
llvm::Value *metadata) {
|
|
auto vwTable = IGF.emitValueWitnessTableRefForMetadata(metadata);
|
|
auto addr = emitAddressOfValueWitnessTableValue(
|
|
IGF, vwTable, ValueWitness::ExtraInhabitantCount);
|
|
IGF.Builder.CreateStore(value, addr);
|
|
}
|
|
|
|
std::pair<llvm::Value *, llvm::Value *>
|
|
irgen::emitLoadOfIsInline(IRGenFunction &IGF, llvm::Value *metadata) {
|
|
auto *flags = emitLoadOfValueWitnessValueFromMetadata(IGF, metadata,
|
|
ValueWitness::Flags);
|
|
auto mask = IGF.IGM.getInt32(ValueWitnessFlags::IsNonInline);
|
|
auto masked = IGF.Builder.CreateAnd(flags, mask);
|
|
return std::make_pair(
|
|
IGF.Builder.CreateICmpEQ(masked, IGF.IGM.getInt32(0),
|
|
flags->getName() + ".isInline"),
|
|
flags);
|
|
}
|
|
|
|
llvm::Value *irgen::emitLoadOfSize(IRGenFunction &IGF, llvm::Value *metadata) {
|
|
auto *size = emitLoadOfValueWitnessValueFromMetadata(IGF, metadata,
|
|
ValueWitness::Size);
|
|
return size;
|
|
}
|
|
|
|
llvm::Value *irgen::emitAlignMaskFromFlags(IRGenFunction &IGF,
|
|
llvm::Value *flags) {
|
|
auto flagsAsSize = IGF.Builder.CreateZExtOrTrunc(flags, IGF.IGM.SizeTy);
|
|
auto *alignMask = IGF.IGM.getSize(Size(ValueWitnessFlags::AlignmentMask));
|
|
return IGF.Builder.CreateAnd(flagsAsSize, alignMask,
|
|
flags->getName() + ".alignmentMask");
|
|
}
|
|
|
|
/// Emit a call to do an 'initializeWithCopy' operation.
|
|
void irgen::emitInitializeWithCopyCall(IRGenFunction &IGF,
|
|
SILType T,
|
|
Address dest,
|
|
Address src) {
|
|
llvm::Value *metadata;
|
|
auto fn = IGF.emitValueWitnessFunctionRef(T, metadata,
|
|
ValueWitness::InitializeWithCopy);
|
|
auto destPtr = emitCastToOpaquePtr(IGF, dest);
|
|
auto srcPtr = emitCastToOpaquePtr(IGF, src);
|
|
IGF.Builder.CreateCall(fn, {destPtr, srcPtr, metadata});
|
|
}
|
|
|
|
llvm::Value *irgen::emitInitializeWithCopyCall(IRGenFunction &IGF,
|
|
llvm::Value *metadata,
|
|
Address dest, Address src) {
|
|
auto copyFn = emitLoadOfValueWitnessFunctionFromMetadata(
|
|
IGF, metadata, ValueWitness::InitializeWithCopy);
|
|
auto destPtr = emitCastToOpaquePtr(IGF, dest);
|
|
auto srcPtr = emitCastToOpaquePtr(IGF, src);
|
|
llvm::CallInst *call = IGF.Builder.CreateCall(
|
|
copyFn, {destPtr, srcPtr, metadata});
|
|
|
|
return call;
|
|
}
|
|
|
|
/// Emit a call to do an 'initializeWithTake' operation.
|
|
void irgen::emitInitializeWithTakeCall(IRGenFunction &IGF,
|
|
SILType T,
|
|
Address dest,
|
|
Address src) {
|
|
llvm::Value *metadata;
|
|
auto fn = IGF.emitValueWitnessFunctionRef(T, metadata,
|
|
ValueWitness::InitializeWithTake);
|
|
auto destPtr = emitCastToOpaquePtr(IGF, dest);
|
|
auto srcPtr = emitCastToOpaquePtr(IGF, src);
|
|
IGF.Builder.CreateCall(fn, {destPtr, srcPtr, metadata});
|
|
}
|
|
|
|
llvm::Value *irgen::emitInitializeWithTakeCall(IRGenFunction &IGF,
|
|
llvm::Value *metadata,
|
|
Address dest, Address src) {
|
|
auto copyFn = emitLoadOfValueWitnessFunctionFromMetadata(
|
|
IGF, metadata, ValueWitness::InitializeWithTake);
|
|
auto destPtr = emitCastToOpaquePtr(IGF, dest);
|
|
auto srcPtr = emitCastToOpaquePtr(IGF, src);
|
|
llvm::CallInst *call =
|
|
IGF.Builder.CreateCall(copyFn, {destPtr, srcPtr, metadata});
|
|
|
|
return call;
|
|
}
|
|
|
|
/// Emit a call to do a 'destroy' operation.
|
|
void irgen::emitDestroyCall(IRGenFunction &IGF,
|
|
SILType T,
|
|
Address object) {
|
|
// If T is a trivial/POD type, nothing needs to be done.
|
|
if (IGF.IGM.getTypeProperties(T).isTrivial())
|
|
return;
|
|
llvm::Value *metadata;
|
|
auto fn = IGF.emitValueWitnessFunctionRef(T, metadata,
|
|
ValueWitness::Destroy);
|
|
auto objectPtr = emitCastToOpaquePtr(IGF, object);
|
|
IGF.Builder.CreateCall(fn, {objectPtr, metadata});
|
|
}
|
|
|
|
void irgen::emitDestroyCall(IRGenFunction &IGF, llvm::Value *metadata,
|
|
Address object) {
|
|
auto fn = emitLoadOfValueWitnessFunctionFromMetadata(IGF, metadata,
|
|
ValueWitness::Destroy);
|
|
auto objectPtr = emitCastToOpaquePtr(IGF, object);
|
|
IGF.Builder.CreateCall(fn, {objectPtr, metadata});
|
|
}
|
|
|
|
static llvm::Constant *getAllocateValueBufferFunction(IRGenModule &IGM) {
|
|
|
|
llvm::Type *argTys[] = {IGM.TypeMetadataPtrTy, IGM.OpaquePtrTy};
|
|
|
|
llvm::SmallString<40> fnName("__swift_allocate_value_buffer");
|
|
|
|
return IGM.getOrCreateHelperFunction(
|
|
fnName, IGM.OpaquePtrTy, argTys,
|
|
[&](IRGenFunction &IGF) {
|
|
auto it = IGF.CurFn->arg_begin();
|
|
auto *metadata = &*(it++);
|
|
auto buffer = Address(&*(it++), IGM.OpaqueTy, Alignment(1));
|
|
|
|
// Dynamically check whether this type is inline or needs an allocation.
|
|
llvm::Value *isInline, *flags;
|
|
std::tie(isInline, flags) = emitLoadOfIsInline(IGF, metadata);
|
|
|
|
auto *outlineBB = IGF.createBasicBlock("outline.allocateValueInBuffer");
|
|
auto *doneBB = IGF.createBasicBlock("done");
|
|
llvm::Value *addressInline, *addressOutline;
|
|
addressInline = buffer.getAddress();
|
|
auto *origBB = IGF.Builder.GetInsertBlock();
|
|
IGF.Builder.CreateCondBr(isInline, doneBB, outlineBB);
|
|
|
|
IGF.Builder.emitBlock(outlineBB);
|
|
{
|
|
auto *size = emitLoadOfSize(IGF, metadata);
|
|
auto *alignMask = emitAlignMaskFromFlags(IGF, flags);
|
|
auto valueAddr =
|
|
IGF.emitAllocRawCall(size, alignMask, "outline.ValueBuffer");
|
|
IGF.Builder.CreateStore(
|
|
valueAddr,
|
|
Address(IGF.Builder.CreateBitCast(buffer.getAddress(), IGM.PtrTy),
|
|
IGM.Int8PtrTy, Alignment(1)));
|
|
addressOutline =
|
|
IGF.Builder.CreateBitCast(valueAddr, IGM.OpaquePtrTy);
|
|
IGF.Builder.CreateBr(doneBB);
|
|
}
|
|
|
|
IGF.Builder.emitBlock(doneBB);
|
|
auto *addressOfValue = IGF.Builder.CreatePHI(IGM.OpaquePtrTy, 2);
|
|
addressOfValue->addIncoming(addressInline, origBB);
|
|
addressOfValue->addIncoming(addressOutline, outlineBB);
|
|
IGF.Builder.CreateRet(addressOfValue);
|
|
},
|
|
true /*noinline*/);
|
|
}
|
|
|
|
Address irgen::emitAllocateValueInBuffer(IRGenFunction &IGF, SILType type,
|
|
Address buffer) {
|
|
// Handle FixedSize types.
|
|
auto &IGM = IGF.IGM;
|
|
auto *storagePtrTy = IGM.PtrTy;
|
|
auto storageTy = IGM.getStorageType(type);
|
|
auto &Builder = IGF.Builder;
|
|
if (auto *fixedTI = dyn_cast<FixedTypeInfo>(&IGF.getTypeInfo(type))) {
|
|
auto packing = fixedTI->getFixedPacking(IGM);
|
|
|
|
// Inline representation.
|
|
if (packing == FixedPacking::OffsetZero) {
|
|
return Address(Builder.CreateBitCast(buffer.getAddress(), storagePtrTy),
|
|
storageTy, buffer.getAlignment());
|
|
}
|
|
|
|
// Outline representation.
|
|
assert(packing == FixedPacking::Allocate && "Expect non dynamic packing");
|
|
auto size = fixedTI->getStaticSize(IGM);
|
|
auto alignMask = fixedTI->getStaticAlignmentMask(IGM);
|
|
auto valueAddr =
|
|
IGF.emitAllocRawCall(size, alignMask, "outline.ValueBuffer");
|
|
Builder.CreateStore(valueAddr,
|
|
Address(Builder.CreateBitCast(buffer.getAddress(),
|
|
IGF.IGM.Int8PtrPtrTy),
|
|
IGM.Int8PtrTy, buffer.getAlignment()));
|
|
return Address(Builder.CreateBitCast(valueAddr, storagePtrTy), storageTy,
|
|
buffer.getAlignment());
|
|
}
|
|
|
|
// Dynamic packing.
|
|
|
|
/// Call a function to handle the non-fixed case.
|
|
auto *allocateFun = getAllocateValueBufferFunction(IGF.IGM);
|
|
auto *fnType = cast<llvm::Function>(allocateFun)->getFunctionType();
|
|
auto *metadata = IGF.emitTypeMetadataRefForLayout(type);
|
|
auto *call = Builder.CreateCall(
|
|
fnType, allocateFun,
|
|
{metadata, Builder.CreateBitCast(buffer.getAddress(), IGM.OpaquePtrTy)});
|
|
call->setCallingConv(IGF.IGM.DefaultCC);
|
|
call->setDoesNotThrow();
|
|
|
|
auto addressOfValue = Builder.CreateBitCast(call, storagePtrTy);
|
|
return Address(addressOfValue, storageTy, Alignment(1));
|
|
}
|
|
|
|
static llvm::Constant *getProjectValueInBufferFunction(IRGenModule &IGM) {
|
|
|
|
llvm::Type *argTys[] = {IGM.TypeMetadataPtrTy, IGM.OpaquePtrTy};
|
|
|
|
llvm::SmallString<40> fnName("__swift_project_value_buffer");
|
|
|
|
return IGM.getOrCreateHelperFunction(
|
|
fnName, IGM.OpaquePtrTy, argTys,
|
|
[&](IRGenFunction &IGF) {
|
|
auto it = IGF.CurFn->arg_begin();
|
|
auto *metadata = &*(it++);
|
|
auto buffer = Address(&*(it++), IGM.OpaqueTy, Alignment(1));
|
|
auto &Builder = IGF.Builder;
|
|
|
|
// Dynamically check whether this type is inline or needs an allocation.
|
|
llvm::Value *isInline, *flags;
|
|
std::tie(isInline, flags) = emitLoadOfIsInline(IGF, metadata);
|
|
|
|
auto *outlineBB = IGF.createBasicBlock("outline.projectValueInBuffer");
|
|
auto *doneBB = IGF.createBasicBlock("done");
|
|
llvm::Value *addressInline, *addressOutline;
|
|
auto *origBB = Builder.GetInsertBlock();
|
|
addressInline = buffer.getAddress();
|
|
|
|
Builder.CreateCondBr(isInline, doneBB, outlineBB);
|
|
|
|
Builder.emitBlock(outlineBB);
|
|
{
|
|
addressOutline = Builder.CreateLoad(
|
|
Address(Builder.CreateBitCast(buffer.getAddress(), IGM.PtrTy),
|
|
IGM.OpaquePtrTy, Alignment(1)));
|
|
Builder.CreateBr(doneBB);
|
|
}
|
|
|
|
Builder.emitBlock(doneBB);
|
|
auto *addressOfValue = Builder.CreatePHI(IGM.OpaquePtrTy, 2);
|
|
addressOfValue->addIncoming(addressInline, origBB);
|
|
addressOfValue->addIncoming(addressOutline, outlineBB);
|
|
|
|
Builder.CreateRet(addressOfValue);
|
|
},
|
|
true /*noinline*/);
|
|
}
|
|
|
|
Address irgen::emitProjectValueInBuffer(IRGenFunction &IGF, SILType type,
|
|
Address buffer) {
|
|
// Handle FixedSize types.
|
|
auto &IGM = IGF.IGM;
|
|
auto *storagePtrTy = IGM.PtrTy;
|
|
auto storageTy = IGM.getStorageType(type);
|
|
auto &Builder = IGF.Builder;
|
|
if (auto *fixedTI = dyn_cast<FixedTypeInfo>(&IGF.getTypeInfo(type))) {
|
|
auto packing = fixedTI->getFixedPacking(IGM);
|
|
|
|
// Inline representation.
|
|
if (packing == FixedPacking::OffsetZero) {
|
|
return Address(Builder.CreateBitCast(buffer.getAddress(), storagePtrTy),
|
|
storageTy, buffer.getAlignment());
|
|
}
|
|
|
|
// Outline representation.
|
|
assert(packing == FixedPacking::Allocate && "Expect non dynamic packing");
|
|
auto valueAddr = Builder.CreateLoad(
|
|
Address(Builder.CreateBitCast(buffer.getAddress(), IGM.PtrTy),
|
|
storagePtrTy, buffer.getAlignment()));
|
|
return Address(Builder.CreateBitCast(valueAddr, storagePtrTy), storageTy,
|
|
buffer.getAlignment());
|
|
}
|
|
|
|
// Dynamic packing.
|
|
auto *projectFun = getProjectValueInBufferFunction(IGF.IGM);
|
|
auto *fnType = cast<llvm::Function>(projectFun)->getFunctionType();
|
|
auto *metadata = IGF.emitTypeMetadataRefForLayout(type);
|
|
auto *call = Builder.CreateCall(
|
|
fnType, projectFun,
|
|
{metadata, Builder.CreateBitCast(buffer.getAddress(), IGM.OpaquePtrTy)});
|
|
call->setCallingConv(IGF.IGM.DefaultCC);
|
|
call->setDoesNotThrow();
|
|
|
|
auto addressOfValue = Builder.CreateBitCast(call, storagePtrTy);
|
|
return Address(addressOfValue, storageTy, Alignment(1));
|
|
}
|
|
|
|
llvm::Value *
|
|
irgen::emitGetEnumTagSinglePayloadGenericCall(IRGenFunction &IGF,
|
|
SILType payloadType,
|
|
const TypeInfo &payloadTI,
|
|
llvm::Value *numExtraCases,
|
|
Address address,
|
|
GetExtraInhabitantTagEmitter emitter) {
|
|
auto getExtraInhabitantTagFn =
|
|
getOrCreateGetExtraInhabitantTagFunction(IGF.IGM, payloadType,
|
|
payloadTI, emitter);
|
|
// Sign the getExtraInhabitantTag function with the C function pointer schema.
|
|
if (auto schema = IGF.IGM.getOptions().PointerAuth.FunctionPointers) {
|
|
if (schema.hasOtherDiscrimination())
|
|
schema = IGF.IGM.getOptions().PointerAuth.GetExtraInhabitantTagFunction;
|
|
getExtraInhabitantTagFn = IGF.IGM.getConstantSignedPointer(
|
|
getExtraInhabitantTagFn, schema, PointerAuthEntity(), nullptr);
|
|
}
|
|
|
|
// We assume this is never a reabstracted type.
|
|
auto type = payloadType.getASTType();
|
|
assert(type->isLegalFormalType());
|
|
auto metadata = IGF.emitTypeMetadataRef(type);
|
|
|
|
auto ptr = IGF.Builder.CreateBitCast(address.getAddress(),
|
|
IGF.IGM.OpaquePtrTy);
|
|
|
|
auto getEnumTagGenericFn =
|
|
IGF.IGM.getGetEnumTagSinglePayloadGenericFunctionPointer();
|
|
auto call = IGF.Builder.CreateCall(getEnumTagGenericFn,
|
|
{ptr,
|
|
numExtraCases,
|
|
metadata,
|
|
getExtraInhabitantTagFn});
|
|
call->setCallingConv(IGF.IGM.SwiftCC);
|
|
return call;
|
|
}
|
|
|
|
llvm::Constant *
|
|
irgen::getOrCreateGetExtraInhabitantTagFunction(IRGenModule &IGM,
|
|
SILType objectType,
|
|
const TypeInfo &objectTI,
|
|
GetExtraInhabitantTagEmitter emitter) {
|
|
// We assume this is never a reabstracted type.
|
|
CanType type = objectType.getASTType();
|
|
assert(type->isLegalFormalType());
|
|
|
|
auto fnTy = llvm::FunctionType::get(IGM.Int32Ty,
|
|
{IGM.OpaquePtrTy,
|
|
IGM.Int32Ty,
|
|
IGM.TypeMetadataPtrTy},
|
|
false);
|
|
|
|
// TODO: use a meaningful mangled name and internal/shared linkage.
|
|
auto fn = llvm::Function::Create(fnTy, llvm::Function::PrivateLinkage,
|
|
"__swift_get_extra_inhabitant_index",
|
|
&IGM.Module);
|
|
fn->setAttributes(IGM.constructInitialAttributes());
|
|
fn->setCallingConv(IGM.SwiftCC);
|
|
IRGenFunction IGF(IGM, fn);
|
|
if (IGM.DebugInfo)
|
|
IGM.DebugInfo->emitArtificialFunction(IGF, fn);
|
|
auto parameters = IGF.collectParameters();
|
|
auto ptr = parameters.claimNext();
|
|
auto xiCount = parameters.claimNext();
|
|
auto metadata = parameters.claimNext();
|
|
|
|
// Bind the metadata to make any archetypes available.
|
|
IGF.bindLocalTypeDataFromTypeMetadata(type, IsExact, metadata,
|
|
MetadataState::Complete);
|
|
|
|
// Form a well-typed address from the opaque pointer.
|
|
ptr = IGF.Builder.CreateBitCast(ptr, IGM.PtrTy);
|
|
Address addr = objectTI.getAddressForPointer(ptr);
|
|
|
|
auto tag = emitter(IGF, addr, xiCount);
|
|
IGF.Builder.CreateRet(tag);
|
|
|
|
return fn;
|
|
}
|
|
|
|
void
|
|
irgen::emitStoreEnumTagSinglePayloadGenericCall(IRGenFunction &IGF,
|
|
SILType payloadType,
|
|
const TypeInfo &payloadTI,
|
|
llvm::Value *whichCase,
|
|
llvm::Value *numExtraCases,
|
|
Address address,
|
|
StoreExtraInhabitantTagEmitter emitter) {
|
|
auto storeExtraInhabitantTagFn =
|
|
getOrCreateStoreExtraInhabitantTagFunction(IGF.IGM, payloadType,
|
|
payloadTI, emitter);
|
|
|
|
// Sign the getExtraInhabitantTag function with the C function pointer schema.
|
|
if (auto schema = IGF.IGM.getOptions().PointerAuth.FunctionPointers) {
|
|
if (schema.hasOtherDiscrimination())
|
|
schema = IGF.IGM.getOptions().PointerAuth.StoreExtraInhabitantTagFunction;
|
|
storeExtraInhabitantTagFn = IGF.IGM.getConstantSignedPointer(
|
|
storeExtraInhabitantTagFn, schema, PointerAuthEntity(), nullptr);
|
|
}
|
|
|
|
// We assume this is never a reabstracted type.
|
|
auto type = payloadType.getASTType();
|
|
assert(type->isLegalFormalType());
|
|
auto metadata = IGF.emitTypeMetadataRef(type);
|
|
|
|
auto ptr = IGF.Builder.CreateBitCast(address.getAddress(),
|
|
IGF.IGM.OpaquePtrTy);
|
|
|
|
auto storeEnumTagGenericFn =
|
|
IGF.IGM.getStoreEnumTagSinglePayloadGenericFunctionPointer();
|
|
auto call = IGF.Builder.CreateCall(storeEnumTagGenericFn,
|
|
{ptr,
|
|
whichCase,
|
|
numExtraCases,
|
|
metadata,
|
|
storeExtraInhabitantTagFn});
|
|
call->setCallingConv(IGF.IGM.SwiftCC);
|
|
}
|
|
|
|
llvm::Constant *
|
|
irgen::getOrCreateStoreExtraInhabitantTagFunction(IRGenModule &IGM,
|
|
SILType objectType,
|
|
const TypeInfo &objectTI,
|
|
StoreExtraInhabitantTagEmitter emitter) {
|
|
// We assume this is never a reabstracted type.
|
|
CanType type = objectType.getASTType();
|
|
assert(type->isLegalFormalType());
|
|
|
|
auto fnTy = llvm::FunctionType::get(IGM.VoidTy,
|
|
{IGM.OpaquePtrTy,
|
|
IGM.Int32Ty,
|
|
IGM.Int32Ty,
|
|
IGM.TypeMetadataPtrTy},
|
|
false);
|
|
|
|
// TODO: use a meaningful mangled name and internal/shared linkage.
|
|
auto fn = llvm::Function::Create(fnTy, llvm::Function::PrivateLinkage,
|
|
"__swift_store_extra_inhabitant_index",
|
|
&IGM.Module);
|
|
fn->setAttributes(IGM.constructInitialAttributes());
|
|
fn->setCallingConv(IGM.SwiftCC);
|
|
IRGenFunction IGF(IGM, fn);
|
|
if (IGM.DebugInfo)
|
|
IGM.DebugInfo->emitArtificialFunction(IGF, fn);
|
|
auto parameters = IGF.collectParameters();
|
|
auto ptr = parameters.claimNext();
|
|
auto tag = parameters.claimNext();
|
|
auto xiCount = parameters.claimNext();
|
|
auto metadata = parameters.claimNext();
|
|
|
|
// Bind the metadata to make any archetypes available.
|
|
IGF.bindLocalTypeDataFromTypeMetadata(type, IsExact, metadata,
|
|
MetadataState::Complete);
|
|
|
|
// Form a well-typed address from the opaque pointer.
|
|
ptr = IGF.Builder.CreateBitCast(ptr, IGM.PtrTy);
|
|
Address addr = objectTI.getAddressForPointer(ptr);
|
|
|
|
emitter(IGF, addr, tag, xiCount);
|
|
IGF.Builder.CreateRetVoid();
|
|
|
|
return fn;
|
|
}
|
|
|
|
llvm::Value *TypeInfo::getExtraInhabitantTagDynamic(IRGenFunction &IGF,
|
|
Address address,
|
|
SILType T,
|
|
llvm::Value *knownXICount,
|
|
bool isOutlined) const {
|
|
if (auto fixedTI = dyn_cast<FixedTypeInfo>(this)) {
|
|
auto index = fixedTI->getExtraInhabitantIndex(IGF, address, T, isOutlined);
|
|
// The runtime APIs expect that 0 means the payload case and 1+
|
|
// means the extra cases, but in IRGen, getExtraInhabitantIndex
|
|
// returns -1 for the payload case and 0+ for extra inhabitants.
|
|
// This is an easy adjustment to make.
|
|
auto one = llvm::ConstantInt::get(IGF.IGM.Int32Ty, 1);
|
|
auto tag = IGF.Builder.CreateAdd(index, one);
|
|
return tag;
|
|
} else {
|
|
if (!knownXICount)
|
|
knownXICount = emitLoadOfExtraInhabitantCount(IGF, T);
|
|
|
|
auto tag = getEnumTagSinglePayload(IGF, /*num extra cases*/ knownXICount,
|
|
address, T, isOutlined);
|
|
return tag;
|
|
}
|
|
}
|
|
|
|
void TypeInfo::storeExtraInhabitantTagDynamic(IRGenFunction &IGF,
|
|
llvm::Value *tag,
|
|
Address address,
|
|
SILType T,
|
|
bool isOutlined) const {
|
|
if (auto fixedTI = dyn_cast<FixedTypeInfo>(this)) {
|
|
// The runtime APIs expect that 0 means the payload case and 1+
|
|
// means the extra cases, but in IRGen, storeExtraInhabitant
|
|
// expects extra inhabitants to be indexed as 0+.
|
|
// This is an easy adjustment to make.
|
|
auto one = llvm::ConstantInt::get(IGF.IGM.Int32Ty, 1);
|
|
auto index = IGF.Builder.CreateSub(tag, one);
|
|
fixedTI->storeExtraInhabitant(IGF, index, address, T, isOutlined);
|
|
} else {
|
|
storeEnumTagSinglePayload(IGF, tag, tag, address, T, isOutlined);
|
|
}
|
|
}
|