averello/swift-mirror
1
0
Fork 0
mirror of https://github.com/apple/swift.git synced 2025-12-14 20:36:38 +01:00
Code Issues Packages Projects Releases Wiki Activity
Files
103630296d3ebcc41d7c48f0ab34b697efbfb51a
swift-mirror/lib/IRGen/GenFunc.cpp
Arnold Schwaighofer 5ab6df8f94 IRGen: Fix capture of indirect values in `[onstack]` closures 
A [onstack] closure does not take ownership of its arguments. It is
therefore not correct to use an initWithTake copy of indirect arguments.

Instead we just capture the address of the indirect argument.

rdar://61261982
2020-04-06 13:23:34 -07:00

1864 lines
75 KiB
C++
Raw Blame History

//===--- GenFunc.cpp - Swift IR Generation for Function Types -------------===//
//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See https://swift.org/LICENSE.txt for license information
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
//
//===----------------------------------------------------------------------===//
//
// This file implements IR generation for function types in Swift. This
// includes creating the IR type as well as capturing variables and
// performing calls.
//
// Swift supports three representations of functions:
//
// - thin, which are just a function pointer;
//
// - thick, which are a pair of a function pointer and
// an optional ref-counted opaque context pointer; and
//
// - block, which match the Apple blocks extension: a ref-counted
// pointer to a mostly-opaque structure with the function pointer
// stored at a fixed offset.
//
// The order of function parameters is as follows:
//
// - indirect return pointer
// - block context parameter, if applicable
// - expanded formal parameter types
// - implicit generic parameters
// - thick context parameter, if applicable
// - error result out-parameter, if applicable
// - witness_method generic parameters, if applicable
//
// The context and error parameters are last because they are
// optional: we'd like to be able to turn a thin function into a
// thick function, or a non-throwing function into a throwing one,
// without adding a thunk. A thick context parameter is required
// (but can be passed undef) if an error result is required.
//
// The additional generic parameters for witness methods follow the
// same logic: we'd like to be able to use non-generic method
// implementations directly as protocol witnesses if the rest of the
// ABI matches up.
//
// Note that some of this business with context parameters and error
// results is just IR formalism; on most of our targets, both of
// these are passed in registers. This is also why passing them
// as the final argument isn't bad for performance.
//
// For now, function pointer types are always stored as opaque
// pointers in LLVM IR; using a well-typed function type is
// very challenging because of issues with recursive type expansion,
// which can potentially introduce infinite types. For example:
// struct A {
// var fn: (A) -> ()
// }
// Our CC lowering expands the fields of A into the argument list
// of A.fn, which is necessarily infinite. Attempting to use better
// types when not in a situation like this would just make the
// compiler complacent, leading to a long tail of undiscovered
// crashes. So instead we always store as i8* and require the
// bitcast whenever we change representations.
//
//===----------------------------------------------------------------------===//
#include "swift/AST/ASTContext.h"
#include "swift/AST/ASTWalker.h"
#include "swift/AST/Builtins.h"
#include "swift/AST/Decl.h"
#include "swift/AST/IRGenOptions.h"
#include "swift/AST/Module.h"
#include "swift/AST/Pattern.h"
#include "swift/AST/PrettyStackTrace.h"
#include "swift/AST/SubstitutionMap.h"
#include "swift/AST/Types.h"
#include "swift/IRGen/Linking.h"
#include "clang/AST/ASTContext.h"
#include "clang/CodeGen/CodeGenABITypes.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/Module.h"
#include "llvm/ProfileData/InstrProf.h"
#include "llvm/Support/Debug.h"
#include "llvm/ADT/StringSwitch.h"
#include "BitPatternBuilder.h"
#include "Callee.h"
#include "ConstantBuilder.h"
#include "EnumPayload.h"
#include "Explosion.h"
#include "FixedTypeInfo.h"
#include "GenCall.h"
#include "GenClass.h"
#include "GenFunc.h"
#include "GenHeap.h"
#include "GenMeta.h"
#include "GenObjC.h"
#include "GenPointerAuth.h"
#include "GenPoly.h"
#include "GenProto.h"
#include "GenType.h"
#include "HeapTypeInfo.h"
#include "IRGenDebugInfo.h"
#include "IRGenFunction.h"
#include "IRGenModule.h"
#include "IndirectTypeInfo.h"
#include "ScalarPairTypeInfo.h"
#include "Signature.h"
#include "IRGenMangler.h"
using namespace swift;
using namespace irgen;
namespace {
/// Information about the IR-level signature of a function type.
class FuncSignatureInfo {
private:
/// The SIL function type being represented.
const CanSILFunctionType FormalType;
mutable Signature TheSignature;
public:
FuncSignatureInfo(CanSILFunctionType formalType)
: FormalType(formalType) {}
Signature getSignature(IRGenModule &IGM) const;
};
/// The @thin function type-info class.
class ThinFuncTypeInfo : public PODSingleScalarTypeInfo<ThinFuncTypeInfo,
LoadableTypeInfo>,
public FuncSignatureInfo {
ThinFuncTypeInfo(CanSILFunctionType formalType, llvm::Type *storageType,
Size size, Alignment align,
const SpareBitVector &spareBits)
: PODSingleScalarTypeInfo(storageType, size, spareBits, align),
FuncSignatureInfo(formalType)
{
}
public:
static const ThinFuncTypeInfo *create(CanSILFunctionType formalType,
llvm::Type *storageType,
Size size, Alignment align,
const SpareBitVector &spareBits) {
return new ThinFuncTypeInfo(formalType, storageType, size, align,
spareBits);
}
TypeLayoutEntry *buildTypeLayoutEntry(IRGenModule &IGM,
SILType T) const override {
return IGM.typeLayoutCache.getOrCreateScalarEntry(*this, T);
}
bool mayHaveExtraInhabitants(IRGenModule &IGM) const override {
return true;
}
unsigned getFixedExtraInhabitantCount(IRGenModule &IGM) const override {
return getFunctionPointerExtraInhabitantCount(IGM);
}
APInt getFixedExtraInhabitantValue(IRGenModule &IGM,
unsigned bits,
unsigned index) const override {
return getFunctionPointerFixedExtraInhabitantValue(IGM, bits, index, 0);
}
llvm::Value *getExtraInhabitantIndex(IRGenFunction &IGF, Address src,
SILType T, bool isOutlined)
const override {
return getFunctionPointerExtraInhabitantIndex(IGF, src);
}
void storeExtraInhabitant(IRGenFunction &IGF, llvm::Value *index,
Address dest, SILType T, bool isOutlined)
const override {
return storeFunctionPointerExtraInhabitant(IGF, index, dest);
}
};
/// The @thick function type-info class.
class FuncTypeInfo :
public ScalarPairTypeInfo<FuncTypeInfo, ReferenceTypeInfo>,
public FuncSignatureInfo {
FuncTypeInfo(CanSILFunctionType formalType, llvm::StructType *storageType,
Size size, Alignment align, SpareBitVector &&spareBits,
IsPOD_t pod)
: ScalarPairTypeInfo(storageType, size, std::move(spareBits), align, pod),
FuncSignatureInfo(formalType)
{
}
public:
static const FuncTypeInfo *create(CanSILFunctionType formalType,
llvm::StructType *storageType,
Size size, Alignment align,
SpareBitVector &&spareBits,
IsPOD_t pod) {
return new FuncTypeInfo(formalType, storageType, size, align,
std::move(spareBits), pod);
}
// Function types do not satisfy allowsOwnership.
#define REF_STORAGE(Name, name, ...) \
const TypeInfo * \
create##Name##StorageType(TypeConverter &TC, \
bool isOptional) const override { \
llvm_unreachable("[" #name "] function type"); \
}
#include "swift/AST/ReferenceStorage.def"
TypeLayoutEntry *buildTypeLayoutEntry(IRGenModule &IGM,
SILType T) const override {
return IGM.typeLayoutCache.getOrCreateScalarEntry(*this, T);
}
static Size getFirstElementSize(IRGenModule &IGM) {
return IGM.getPointerSize();
}
static StringRef getFirstElementLabel() {
return ".fn";
}
static bool isFirstElementTrivial() {
return true;
}
void emitRetainFirstElement(IRGenFunction &IGF, llvm::Value *fn,
Optional<Atomicity> atomicity = None) const {}
void emitReleaseFirstElement(IRGenFunction &IGF, llvm::Value *fn,
Optional<Atomicity> atomicity = None) const {}
void emitAssignFirstElement(IRGenFunction &IGF, llvm::Value *fn,
Address fnAddr) const {
IGF.Builder.CreateStore(fn, fnAddr);
}
static Size getSecondElementOffset(IRGenModule &IGM) {
return IGM.getPointerSize();
}
static Size getSecondElementSize(IRGenModule &IGM) {
return IGM.getPointerSize();
}
static StringRef getSecondElementLabel() {
return ".data";
}
bool isSecondElementTrivial() const {
return isPOD(ResilienceExpansion::Maximal);
}
void emitRetainSecondElement(IRGenFunction &IGF, llvm::Value *data,
Optional<Atomicity> atomicity = None) const {
if (!isPOD(ResilienceExpansion::Maximal)) {
if (!atomicity) atomicity = IGF.getDefaultAtomicity();
IGF.emitNativeStrongRetain(data, *atomicity);
}
}
void emitReleaseSecondElement(IRGenFunction &IGF, llvm::Value *data,
Optional<Atomicity> atomicity = None) const {
if (!isPOD(ResilienceExpansion::Maximal)) {
if (!atomicity) atomicity = IGF.getDefaultAtomicity();
IGF.emitNativeStrongRelease(data, *atomicity);
}
}
void emitAssignSecondElement(IRGenFunction &IGF, llvm::Value *context,
Address dataAddr) const {
if (isPOD(ResilienceExpansion::Maximal))
IGF.Builder.CreateStore(context, dataAddr);
else
IGF.emitNativeStrongAssign(context, dataAddr);
}
Address projectFunction(IRGenFunction &IGF, Address address) const {
return projectFirstElement(IGF, address);
}
Address projectData(IRGenFunction &IGF, Address address) const {
return IGF.Builder.CreateStructGEP(address, 1, IGF.IGM.getPointerSize(),
address->getName() + ".data");
}
void strongRetain(IRGenFunction &IGF, Explosion &e,
Atomicity atomicity) const override {
e.claimNext();
emitRetainSecondElement(IGF, e.claimNext(), atomicity);
}
void strongRelease(IRGenFunction &IGF, Explosion &e,
Atomicity atomicity) const override {
e.claimNext();
emitReleaseSecondElement(IGF, e.claimNext(), atomicity);
}
#define NEVER_LOADABLE_CHECKED_REF_STORAGE(Name, name, ...) \
void name##LoadStrong(IRGenFunction &IGF, Address src, \
Explosion &out, bool isOptional) const override { \
llvm_unreachable(#name " references to functions are not supported"); \
} \
void name##TakeStrong(IRGenFunction &IGF, Address src, \
Explosion &out, bool isOptional) const override { \
llvm_unreachable(#name " references to functions are not supported"); \
} \
void name##Init(IRGenFunction &IGF, Explosion &in, \
Address dest, bool isOptional) const override { \
llvm_unreachable(#name " references to functions are not supported"); \
} \
void name##Assign(IRGenFunction &IGF, Explosion &in, \
Address dest, bool isOptional) const override { \
llvm_unreachable(#name " references to functions are not supported"); \
}
#define ALWAYS_LOADABLE_CHECKED_REF_STORAGE(Name, name, ...) \
void strongRetain##Name(IRGenFunction &IGF, Explosion &e, \
Atomicity atomicity) const override { \
llvm_unreachable(#name " references to functions are not supported"); \
} \
void strongRetain##Name##Release(IRGenFunction &IGF, \
Explosion &e, \
Atomicity atomicity) const override { \
llvm_unreachable(#name " references to functions are not supported"); \
} \
void name##Retain(IRGenFunction &IGF, Explosion &e, \
Atomicity atomicity) const override { \
llvm_unreachable(#name " references to functions are not supported"); \
} \
void name##Release(IRGenFunction &IGF, Explosion &e, \
Atomicity atomicity) const override { \
llvm_unreachable(#name " references to functions are not supported"); \
}
#define SOMETIMES_LOADABLE_CHECKED_REF_STORAGE(Name, name, ...) \
NEVER_LOADABLE_CHECKED_REF_STORAGE(Name, name, "...") \
ALWAYS_LOADABLE_CHECKED_REF_STORAGE(Name, name, "...")
#include "swift/AST/ReferenceStorage.def"
bool mayHaveExtraInhabitants(IRGenModule &IGM) const override {
return true;
}
unsigned getFixedExtraInhabitantCount(IRGenModule &IGM) const override {
return getFunctionPointerExtraInhabitantCount(IGM);
}
APInt getFixedExtraInhabitantValue(IRGenModule &IGM,
unsigned bits,
unsigned index) const override {
return getFunctionPointerFixedExtraInhabitantValue(IGM, bits, index, 0);
}
llvm::Value *getExtraInhabitantIndex(IRGenFunction &IGF, Address src,
SILType T, bool isOutlined)
const override {
src = projectFunction(IGF, src);
return getFunctionPointerExtraInhabitantIndex(IGF, src);
}
APInt getFixedExtraInhabitantMask(IRGenModule &IGM) const override {
// Only the function pointer value is used for extra inhabitants.
auto pointerSize = IGM.getPointerSize();
auto mask = BitPatternBuilder(IGM.Triple.isLittleEndian());
mask.appendSetBits(pointerSize.getValueInBits());
mask.appendClearBits(pointerSize.getValueInBits());
return mask.build().getValue();
}
void storeExtraInhabitant(IRGenFunction &IGF, llvm::Value *index,
Address dest, SILType T, bool isOutlined)
const override {
dest = projectFunction(IGF, dest);
return storeFunctionPointerExtraInhabitant(IGF, index, dest);
}
};
/// The type-info class for ObjC blocks, which are represented by an ObjC
/// heap pointer.
class BlockTypeInfo : public HeapTypeInfo<BlockTypeInfo>,
public FuncSignatureInfo
{
public:
BlockTypeInfo(CanSILFunctionType ty,
llvm::PointerType *storageType,
Size size, SpareBitVector spareBits, Alignment align)
: HeapTypeInfo(storageType, size, spareBits, align),
FuncSignatureInfo(ty)
{
}
ReferenceCounting getReferenceCounting() const {
return ReferenceCounting::Block;
}
TypeLayoutEntry *buildTypeLayoutEntry(IRGenModule &IGM,
SILType T) const override {
return IGM.typeLayoutCache.getOrCreateScalarEntry(*this, T);
}
};
/// The type info class for the on-stack representation of an ObjC block.
///
/// TODO: May not be fixed-layout if we capture generics.
class BlockStorageTypeInfo final
: public IndirectTypeInfo<BlockStorageTypeInfo, FixedTypeInfo>
{
Size CaptureOffset;
public:
BlockStorageTypeInfo(llvm::Type *type, Size size, Alignment align,
SpareBitVector &&spareBits,
IsPOD_t pod, IsBitwiseTakable_t bt, Size captureOffset)
: IndirectTypeInfo(type, size, std::move(spareBits), align, pod, bt,
IsFixedSize),
CaptureOffset(captureOffset)
{}
TypeLayoutEntry *buildTypeLayoutEntry(IRGenModule &IGM,
SILType T) const override {
return IGM.typeLayoutCache.getOrCreateScalarEntry(*this, T);
}
// The lowered type should be an LLVM struct comprising the block header
// (IGM.ObjCBlockStructTy) as its first element and the capture as its
// second.
Address projectBlockHeader(IRGenFunction &IGF, Address storage) const {
return IGF.Builder.CreateStructGEP(storage, 0, Size(0));
}
Address projectCapture(IRGenFunction &IGF, Address storage) const {
return IGF.Builder.CreateStructGEP(storage, 1, CaptureOffset);
}
// TODO
// The frontend will currently never emit copy_addr or destroy_addr for
// block storage.
void assignWithCopy(IRGenFunction &IGF, Address dest, Address src,
SILType T, bool isOutlined) const override {
IGF.unimplemented(SourceLoc(), "copying @block_storage");
}
void initializeWithCopy(IRGenFunction &IGF, Address dest, Address src,
SILType T, bool isOutlined) const override {
IGF.unimplemented(SourceLoc(), "copying @block_storage");
}
void destroy(IRGenFunction &IGF, Address addr, SILType T,
bool isOutlined) const override {
IGF.unimplemented(SourceLoc(), "destroying @block_storage");
}
};
} // end anonymous namespace
const TypeInfo *TypeConverter::convertBlockStorageType(SILBlockStorageType *T) {
// The block storage consists of the block header (ObjCBlockStructTy)
// followed by the lowered type of the capture.
auto &capture = IGM.getTypeInfoForLowered(T->getCaptureType());
// TODO: Support dynamic-sized captures.
const auto *fixedCapture = dyn_cast<FixedTypeInfo>(&capture);
llvm::Type *fixedCaptureTy;
// The block header is pointer aligned. The capture may be worse aligned.
Alignment align = IGM.getPointerAlignment();
Size captureOffset(
IGM.DataLayout.getStructLayout(IGM.ObjCBlockStructTy)->getSizeInBytes());
auto spareBits = BitPatternBuilder(IGM.Triple.isLittleEndian());
spareBits.appendClearBits(captureOffset.getValueInBits());
Size size = captureOffset;
IsPOD_t pod = IsNotPOD;
IsBitwiseTakable_t bt = IsNotBitwiseTakable;
if (!fixedCapture) {
IGM.unimplemented(SourceLoc(), "dynamic @block_storage capture");
fixedCaptureTy = llvm::StructType::get(IGM.getLLVMContext(), {});
} else {
fixedCaptureTy = cast<FixedTypeInfo>(capture).getStorageType();
align = std::max(align, fixedCapture->getFixedAlignment());
captureOffset = captureOffset.roundUpToAlignment(align);
spareBits.padWithSetBitsTo(captureOffset.getValueInBits());
spareBits.append(fixedCapture->getSpareBits());
size = captureOffset + fixedCapture->getFixedSize();
pod = fixedCapture->isPOD(ResilienceExpansion::Maximal);
bt = fixedCapture->isBitwiseTakable(ResilienceExpansion::Maximal);
}
llvm::Type *storageElts[] = {
IGM.ObjCBlockStructTy,
fixedCaptureTy,
};
auto storageTy = llvm::StructType::get(IGM.getLLVMContext(), storageElts,
/*packed*/ false);
return new BlockStorageTypeInfo(storageTy, size, align, spareBits.build(),
pod, bt, captureOffset);
}
Address irgen::projectBlockStorageCapture(IRGenFunction &IGF,
Address storageAddr,
CanSILBlockStorageType storageTy) {
auto &tl = IGF.getTypeInfoForLowered(storageTy).as<BlockStorageTypeInfo>();
return tl.projectCapture(IGF, storageAddr);
}
const TypeInfo *TypeConverter::convertFunctionType(SILFunctionType *T) {
// Handle `@differentiable` and `@differentiable(linear)` functions.
switch (T->getDifferentiabilityKind()) {
case DifferentiabilityKind::Normal:
return convertNormalDifferentiableFunctionType(T);
case DifferentiabilityKind::Linear:
return convertLinearDifferentiableFunctionType(T);
case DifferentiabilityKind::NonDifferentiable:
break;
}
switch (T->getRepresentation()) {
case SILFunctionType::Representation::Block:
return new BlockTypeInfo(CanSILFunctionType(T),
IGM.ObjCBlockPtrTy,
IGM.getPointerSize(),
IGM.getHeapObjectSpareBits(),
IGM.getPointerAlignment());
case SILFunctionType::Representation::Thin:
case SILFunctionType::Representation::Method:
case SILFunctionType::Representation::WitnessMethod:
case SILFunctionType::Representation::ObjCMethod:
case SILFunctionType::Representation::CFunctionPointer:
case SILFunctionType::Representation::Closure:
return ThinFuncTypeInfo::create(CanSILFunctionType(T),
IGM.FunctionPtrTy,
IGM.getPointerSize(),
IGM.getPointerAlignment(),
IGM.getFunctionPointerSpareBits());
case SILFunctionType::Representation::Thick: {
SpareBitVector spareBits;
spareBits.append(IGM.getFunctionPointerSpareBits());
// Although the context pointer of a closure (at least, an escaping one)
// is a refcounted pointer, we'd like to reserve the right to pack small
// contexts into the pointer value, so let's not take any spare bits from
// it.
spareBits.appendClearBits(IGM.getPointerSize().getValueInBits());
if (T->isNoEscape()) {
// @noescape thick functions are trivial types.
return FuncTypeInfo::create(
CanSILFunctionType(T), IGM.NoEscapeFunctionPairTy,
IGM.getPointerSize() * 2, IGM.getPointerAlignment(),
std::move(spareBits), IsPOD);
}
return FuncTypeInfo::create(
CanSILFunctionType(T), IGM.FunctionPairTy, IGM.getPointerSize() * 2,
IGM.getPointerAlignment(), std::move(spareBits), IsNotPOD);
}
}
llvm_unreachable("bad function type representation");
}
Signature FuncSignatureInfo::getSignature(IRGenModule &IGM) const {
// If it's already been filled in, we're done.
if (TheSignature.isValid())
return TheSignature;
// Update the cache and return.
TheSignature = Signature::getUncached(IGM, FormalType);
assert(TheSignature.isValid());
return TheSignature;
}
static const FuncSignatureInfo &
getFuncSignatureInfoForLowered(IRGenModule &IGM, CanSILFunctionType type) {
auto &ti = IGM.getTypeInfoForLowered(type);
switch (type->getRepresentation()) {
case SILFunctionType::Representation::Block:
return ti.as<BlockTypeInfo>();
case SILFunctionType::Representation::Thin:
case SILFunctionType::Representation::CFunctionPointer:
case SILFunctionType::Representation::Method:
case SILFunctionType::Representation::WitnessMethod:
case SILFunctionType::Representation::ObjCMethod:
case SILFunctionType::Representation::Closure:
return ti.as<ThinFuncTypeInfo>();
case SILFunctionType::Representation::Thick:
return ti.as<FuncTypeInfo>();
}
llvm_unreachable("bad function type representation");
}
Signature
IRGenModule::getSignature(CanSILFunctionType type) {
auto &sigInfo = getFuncSignatureInfoForLowered(*this, type);
return sigInfo.getSignature(*this);
}
llvm::FunctionType *
IRGenModule::getFunctionType(CanSILFunctionType type,
llvm::AttributeList &attrs,
ForeignFunctionInfo *foreignInfo) {
auto &sigInfo = getFuncSignatureInfoForLowered(*this, type);
Signature sig = sigInfo.getSignature(*this);
attrs = sig.getAttributes();
if (foreignInfo) *foreignInfo = sig.getForeignInfo();
return sig.getType();
}
ForeignFunctionInfo
IRGenModule::getForeignFunctionInfo(CanSILFunctionType type) {
if (type->getLanguage() == SILFunctionLanguage::Swift)
return ForeignFunctionInfo();
auto &sigInfo = getFuncSignatureInfoForLowered(*this, type);
return sigInfo.getSignature(*this).getForeignInfo();
}
static void emitApplyArgument(IRGenFunction &IGF,
CanSILFunctionType origFnTy,
SILParameterInfo origParam,
CanSILFunctionType substFnTy,
SILParameterInfo substParam,
Explosion &in,
Explosion &out) {
auto silConv = IGF.IGM.silConv;
bool isSubstituted =
(silConv.getSILType(substParam, substFnTy)
!= silConv.getSILType(origParam, origFnTy));
// For indirect arguments, we just need to pass a pointer.
if (silConv.isSILIndirect(origParam)) {
// This address is of the substituted type.
auto addr = in.claimNext();
// If a substitution is in play, just bitcast the address.
if (isSubstituted) {
auto origType =
IGF.IGM.getStoragePointerType(silConv.getSILType(origParam, origFnTy));
addr = IGF.Builder.CreateBitCast(addr, origType);
}
out.add(addr);
return;
}
assert(!silConv.isSILIndirect(origParam)
&& "Unexpected opaque apply parameter.");
// Otherwise, it's an explosion, which we may need to translate,
// both in terms of explosion level and substitution levels.
// Handle the last unsubstituted case.
if (!isSubstituted) {
auto &substArgTI = cast<LoadableTypeInfo>(
IGF.getTypeInfo(silConv.getSILType(substParam, substFnTy)));
substArgTI.reexplode(IGF, in, out);
return;
}
reemitAsUnsubstituted(IGF, silConv.getSILType(origParam, origFnTy),
silConv.getSILType(substParam, substFnTy), in, out);
}
static CanType getArgumentLoweringType(CanType type, SILParameterInfo paramInfo,
bool isNoEscape) {
switch (paramInfo.getConvention()) {
// Capture value parameters by value, consuming them.
case ParameterConvention::Direct_Owned:
case ParameterConvention::Direct_Unowned:
case ParameterConvention::Direct_Guaranteed:
return type;
// Capture indirect parameters if the closure is not [onstack]. [onstack]
// closures don't take ownership of their arguments so we just capture the
// address.
case ParameterConvention::Indirect_In:
case ParameterConvention::Indirect_In_Constant:
case ParameterConvention::Indirect_In_Guaranteed:
if (isNoEscape)
return CanInOutType::get(type);
else
return type;
// Capture inout parameters by pointer.
case ParameterConvention::Indirect_Inout:
case ParameterConvention::Indirect_InoutAliasable:
return CanInOutType::get(type);
}
llvm_unreachable("unhandled convention");
}
static bool isABIIgnoredParameterWithoutStorage(IRGenModule &IGM,
IRGenFunction &IGF,
CanSILFunctionType substType,
unsigned paramIdx) {
auto param = substType->getParameters()[paramIdx];
if (param.isFormalIndirect())
return false;
SILType argType = IGM.silConv.getSILType(param, substType);
auto &ti = IGF.getTypeInfoForLowered(argType.getASTType());
// Empty values don't matter.
return ti.getSchema().empty();
}
/// Find the parameter index for the one (assuming there was only one) partially
/// applied argument ignoring empty types that are not passed as part of the
/// ABI.
static unsigned findSinglePartiallyAppliedParameterIndexIgnoringEmptyTypes(
IRGenFunction &IGF, CanSILFunctionType substType,
CanSILFunctionType outType) {
auto substParameters = substType->getParameters();
auto outParamters = outType->getParameters();
unsigned firstNonEmpty = -1U;
for (unsigned paramIdx = outParamters.size() ; paramIdx != substParameters.size(); ++paramIdx) {
bool isEmpty =
isABIIgnoredParameterWithoutStorage(IGF.IGM, IGF, substType, paramIdx);
assert((isEmpty || firstNonEmpty == -1U) && "Expect at most one partially "
"applied that is passed as an "
"ABI argument");
if (!isEmpty)
firstNonEmpty = paramIdx;
}
assert(firstNonEmpty != -1U);
return firstNonEmpty;
}
/// Emit the forwarding stub function for a partial application.
///
/// If 'layout' is null, there is a single captured value of
/// Swift-refcountable type that is being used directly as the
/// context object.
static llvm::Function *emitPartialApplicationForwarder(IRGenModule &IGM,
const Optional<FunctionPointer> &staticFnPtr,
bool calleeHasContext,
const Signature &origSig,
CanSILFunctionType origType,
CanSILFunctionType substType,
CanSILFunctionType outType,
SubstitutionMap subs,
HeapLayout const *layout,
ArrayRef<ParameterConvention> conventions) {
auto outSig = IGM.getSignature(outType);
llvm::AttributeList outAttrs = outSig.getAttributes();
llvm::FunctionType *fwdTy = outSig.getType();
SILFunctionConventions outConv(outType, IGM.getSILModule());
StringRef FnName;
if (staticFnPtr)
FnName = staticFnPtr->getPointer()->getName();
IRGenMangler Mangler;
std::string thunkName = Mangler.manglePartialApplyForwarder(FnName);
// FIXME: Maybe cache the thunk by function and closure types?.
llvm::Function *fwd =
llvm::Function::Create(fwdTy, llvm::Function::InternalLinkage,
llvm::StringRef(thunkName), &IGM.Module);
fwd->setCallingConv(outSig.getCallingConv());
fwd->setAttributes(outAttrs);
// Merge initial attributes with outAttrs.
llvm::AttrBuilder b;
IGM.constructInitialFnAttributes(b);
fwd->addAttributes(llvm::AttributeList::FunctionIndex, b);
IRGenFunction subIGF(IGM, fwd);
if (IGM.DebugInfo)
IGM.DebugInfo->emitArtificialFunction(subIGF, fwd);
Explosion origParams = subIGF.collectParameters();
// Create a new explosion for potentially reabstracted parameters.
Explosion args;
Address resultValueAddr;
{
// Lower the forwarded arguments in the original function's generic context.
GenericContextScope scope(IGM, origType->getInvocationGenericSignature());
SILFunctionConventions origConv(origType, IGM.getSILModule());
auto &outResultTI = IGM.getTypeInfo(outConv.getSILResultType());
auto &nativeResultSchema = outResultTI.nativeReturnValueSchema(IGM);
auto &origResultTI = IGM.getTypeInfo(origConv.getSILResultType());
auto &origNativeSchema = origResultTI.nativeReturnValueSchema(IGM);
// Forward the indirect return values. We might have to reabstract the
// return value.
if (nativeResultSchema.requiresIndirect()) {
assert(origNativeSchema.requiresIndirect());
auto resultAddr = origParams.claimNext();
resultAddr = subIGF.Builder.CreateBitCast(
resultAddr, IGM.getStoragePointerType(origConv.getSILResultType()));
args.add(resultAddr);
} else if (origNativeSchema.requiresIndirect()) {
assert(!nativeResultSchema.requiresIndirect());
auto stackAddr = outResultTI.allocateStack(
subIGF, outConv.getSILResultType(), "return.temp");
resultValueAddr = stackAddr.getAddress();
auto resultAddr = subIGF.Builder.CreateBitCast(
resultValueAddr,
IGM.getStoragePointerType(origConv.getSILResultType()));
args.add(resultAddr.getAddress());
}
for (auto resultType : origConv.getIndirectSILResultTypes()) {
auto addr = origParams.claimNext();
addr = subIGF.Builder.CreateBitCast(
addr, IGM.getStoragePointerType(resultType));
args.add(addr);
}
// Reemit the parameters as unsubstituted.
for (unsigned i = 0; i < outType->getParameters().size(); ++i) {
auto origParamInfo = origType->getParameters()[i];
auto &ti = IGM.getTypeInfoForLowered(
origParamInfo.getArgumentType(IGM.getSILModule(), origType));
auto schema = ti.getSchema();
auto origParamSILType = IGM.silConv.getSILType(origParamInfo, origType);
// Forward the address of indirect value params.
auto &nativeSchemaOrigParam = ti.nativeParameterValueSchema(IGM);
bool isIndirectParam = origConv.isSILIndirect(origParamInfo);
if (!isIndirectParam && nativeSchemaOrigParam.requiresIndirect()) {
auto addr = origParams.claimNext();
if (addr->getType() != ti.getStorageType()->getPointerTo())
addr = subIGF.Builder.CreateBitCast(addr,
ti.getStorageType()->getPointerTo());
args.add(addr);
continue;
}
auto outTypeParamInfo = outType->getParameters()[i];
// Indirect parameters need no mapping through the native calling
// convention.
if (isIndirectParam) {
emitApplyArgument(subIGF,
origType,
origParamInfo,
outType,
outTypeParamInfo,
origParams, args);
continue;
}
// Map from the native calling convention into the explosion schema.
auto outTypeParamSILType = IGM.silConv.getSILType(origParamInfo, origType);
auto &nativeSchemaOutTypeParam =
IGM.getTypeInfo(outTypeParamSILType).nativeParameterValueSchema(IGM);
Explosion nativeParam;
origParams.transferInto(nativeParam, nativeSchemaOutTypeParam.size());
bindPolymorphicParameter(subIGF, origType, substType, nativeParam, i);
Explosion nonNativeParam = nativeSchemaOutTypeParam.mapFromNative(
subIGF.IGM, subIGF, nativeParam, outTypeParamSILType);
assert(nativeParam.empty());
// Emit unsubstituted argument for call.
Explosion nonNativeApplyArg;
emitApplyArgument(subIGF,
origType, origParamInfo,
outType, outTypeParamInfo,
nonNativeParam,
nonNativeApplyArg);
assert(nonNativeParam.empty());
// Map back from the explosion scheme to the native calling convention for
// the call.
Explosion nativeApplyArg = nativeSchemaOrigParam.mapIntoNative(
subIGF.IGM, subIGF, nonNativeApplyArg, origParamSILType, false);
assert(nonNativeApplyArg.empty());
nativeApplyArg.transferInto(args, nativeApplyArg.size());
}
}
struct AddressToDeallocate {
SILType Type;
const TypeInfo &TI;
StackAddress Addr;
};
SmallVector<AddressToDeallocate, 4> addressesToDeallocate;
bool dependsOnContextLifetime = false;
bool consumesContext;
bool needsAllocas = false;
switch (outType->getCalleeConvention()) {
case ParameterConvention::Direct_Owned:
consumesContext = true;
break;
case ParameterConvention::Direct_Unowned:
case ParameterConvention::Direct_Guaranteed:
consumesContext = false;
break;
case ParameterConvention::Indirect_Inout:
case ParameterConvention::Indirect_InoutAliasable:
case ParameterConvention::Indirect_In:
case ParameterConvention::Indirect_In_Constant:
case ParameterConvention::Indirect_In_Guaranteed:
llvm_unreachable("indirect callables not supported");
}
// Lower the captured arguments in the original function's generic context.
GenericContextScope scope(IGM, origType->getInvocationGenericSignature());
// This is where the context parameter appears.
llvm::Value *rawData = nullptr;
Address data;
unsigned nextCapturedField = 0;
if (!layout) {
rawData = origParams.claimNext();
} else if (!layout->isKnownEmpty()) {
rawData = origParams.claimNext();
data = layout->emitCastTo(subIGF, rawData);
// Restore type metadata bindings, if we have them.
if (layout->hasBindings()) {
auto bindingLayout = layout->getElement(nextCapturedField++);
// The bindings should be fixed-layout inside the object, so we can
// pass None here. If they weren't, we'd have a chicken-egg problem.
auto bindingsAddr = bindingLayout.project(subIGF, data, /*offsets*/ None);
layout->getBindings().restore(subIGF, bindingsAddr,
MetadataState::Complete);
}
// There's still a placeholder to claim if the target type is thick
// or there's an error result.
} else if (outType->getRepresentation()==SILFunctionTypeRepresentation::Thick
|| outType->hasErrorResult()) {
llvm::Value *contextPtr = origParams.claimNext(); (void)contextPtr;
assert(contextPtr->getType() == IGM.RefCountedPtrTy);
}
Explosion polyArgs;
// Emit the polymorphic arguments.
assert((subs.hasAnySubstitutableParams()
== hasPolymorphicParameters(origType) ||
(!subs.hasAnySubstitutableParams() && origType->getRepresentation() ==
SILFunctionTypeRepresentation::WitnessMethod))
&& "should have substitutions iff original function is generic");
WitnessMetadata witnessMetadata;
// If we have a layout we might have to bind polymorphic arguments from the
// captured arguments which we will do later. Otherwise, we have to
// potentially bind polymorphic arguments from the context if it was a
// partially applied argument.
bool hasPolymorphicParams = hasPolymorphicParameters(origType);
if (!layout && hasPolymorphicParams) {
assert(conventions.size() == 1);
// We could have either partially applied an argument from the function
// signature or otherwise we could have a closure context to forward. We only
// care for the former for the purpose of reconstructing polymorphic
// parameters from regular arguments.
if (!calleeHasContext) {
unsigned paramI =
findSinglePartiallyAppliedParameterIndexIgnoringEmptyTypes(
subIGF, substType, outType);
auto paramInfo = substType->getParameters()[paramI];
auto &ti = IGM.getTypeInfoForLowered(
paramInfo.getArgumentType(IGM.getSILModule(), substType));
Explosion param;
auto ref = rawData;
// We can get a '{ swift.refcounted* }' type for AnyObject on linux.
if (!ti.getStorageType()->isPointerTy() &&
ti.isSingleSwiftRetainablePointer(ResilienceExpansion::Maximal))
ref = subIGF.coerceValue(rawData, ti.getStorageType(),
subIGF.IGM.DataLayout);
else
ref = subIGF.Builder.CreateBitCast(rawData, ti.getStorageType());
param.add(ref);
bindPolymorphicParameter(subIGF, origType, substType, param, paramI);
(void)param.claimAll();
}
emitPolymorphicArguments(subIGF, origType, subs,
&witnessMetadata, polyArgs);
}
auto haveContextArgument =
calleeHasContext || hasSelfContextParameter(origType);
// Witness method calls expect self, followed by the self type followed by,
// the witness table at the end of the parameter list. But polymorphic
// arguments come before this.
bool isWitnessMethodCallee = origType->getRepresentation() ==
SILFunctionTypeRepresentation::WitnessMethod;
Explosion witnessMethodSelfValue;
llvm::Value *lastCapturedFieldPtr = nullptr;
// If there's a data pointer required, but it's a swift-retainable
// value being passed as the context, just forward it down.
if (!layout) {
assert(conventions.size() == 1);
// We need to retain the parameter if:
// - we received at +0 (either) and are passing as owned
// - we received as unowned and are passing as guaranteed
auto argConvention = conventions[nextCapturedField++];
switch (argConvention) {
case ParameterConvention::Indirect_In:
case ParameterConvention::Indirect_In_Constant:
case ParameterConvention::Direct_Owned:
if (!consumesContext) subIGF.emitNativeStrongRetain(rawData, subIGF.getDefaultAtomicity());
break;
case ParameterConvention::Indirect_In_Guaranteed:
case ParameterConvention::Direct_Guaranteed:
dependsOnContextLifetime = true;
if (outType->getCalleeConvention() ==
ParameterConvention::Direct_Unowned) {
subIGF.emitNativeStrongRetain(rawData, subIGF.getDefaultAtomicity());
consumesContext = true;
}
break;
case ParameterConvention::Direct_Unowned:
// Make sure we release later if we received at +1.
if (consumesContext)
dependsOnContextLifetime = true;
break;
case ParameterConvention::Indirect_Inout:
case ParameterConvention::Indirect_InoutAliasable:
llvm_unreachable("should never happen!");
}
// FIXME: The naming and documentation here isn't ideal. This
// parameter is always present which is evident since we always
// grab a type to cast to, but sometimes after the polymorphic
// arguments. This is just following the lead of existing (and not
// terribly easy to follow) code.
// If there is a context argument, it comes after the polymorphic
// arguments.
auto argIndex = args.size();
if (haveContextArgument)
argIndex += polyArgs.size();
llvm::Type *expectedArgTy = origSig.getType()->getParamType(argIndex);
llvm::Value *argValue;
if (isIndirectFormalParameter(argConvention)) {
// We can use rawData's type for the alloca because it is a swift
// retainable value. Defensively, give it that type. We can't use the
// expectedArgType because it might be a generic parameter and therefore
// have opaque storage.
auto RetainableValue = rawData;
if (RetainableValue->getType() != subIGF.IGM.RefCountedPtrTy)
RetainableValue = subIGF.Builder.CreateBitCast(
RetainableValue, subIGF.IGM.RefCountedPtrTy);
needsAllocas = true;
auto temporary = subIGF.createAlloca(RetainableValue->getType(),
subIGF.IGM.getPointerAlignment(),
"partial-apply.context");
subIGF.Builder.CreateStore(RetainableValue, temporary);
argValue = temporary.getAddress();
argValue = subIGF.Builder.CreateBitCast(argValue, expectedArgTy);
} else {
argValue = subIGF.Builder.CreateBitCast(rawData, expectedArgTy);
}
args.add(argValue);
// If there's a data pointer required, grab it and load out the
// extra, previously-curried parameters.
} else {
unsigned origParamI = outType->getParameters().size();
assert(layout->getElements().size() == conventions.size()
&& "conventions don't match context layout");
// Calculate non-fixed field offsets.
HeapNonFixedOffsets offsets(subIGF, *layout);
// Perform the loads.
for (unsigned n = layout->getElements().size();
nextCapturedField < n;
++nextCapturedField) {
auto &fieldLayout = layout->getElement(nextCapturedField);
auto &fieldTy = layout->getElementTypes()[nextCapturedField];
auto fieldConvention = conventions[nextCapturedField];
Address fieldAddr = fieldLayout.project(subIGF, data, offsets);
auto &fieldTI = fieldLayout.getType();
lastCapturedFieldPtr = fieldAddr.getAddress();
Explosion param;
switch (fieldConvention) {
case ParameterConvention::Indirect_In:
case ParameterConvention::Indirect_In_Constant: {
auto initStackCopy = [&addressesToDeallocate, &needsAllocas, &param,
&subIGF](const TypeInfo &fieldTI, SILType fieldTy,
Address fieldAddr) {
// The +1 argument is passed indirectly, so we need to copy into a
// temporary.
needsAllocas = true;
auto stackAddr = fieldTI.allocateStack(subIGF, fieldTy, "arg.temp");
auto addressPointer = stackAddr.getAddress().getAddress();
fieldTI.initializeWithCopy(subIGF, stackAddr.getAddress(), fieldAddr,
fieldTy, false);
param.add(addressPointer);
// Remember to deallocate later.
addressesToDeallocate.push_back(
AddressToDeallocate{fieldTy, fieldTI, stackAddr});
};
if (outType->isNoEscape()) {
// If the closure is [onstack] it only captured the address of the
// value. Load that address from the context.
Explosion addressExplosion;
cast<LoadableTypeInfo>(fieldTI).loadAsCopy(subIGF, fieldAddr,
addressExplosion);
assert(fieldTy.isAddress());
auto newFieldTy = fieldTy.getObjectType();
auto &newFieldTI =
subIGF.getTypeInfoForLowered(newFieldTy.getASTType());
fieldAddr =
newFieldTI.getAddressForPointer(addressExplosion.claimNext());
initStackCopy(newFieldTI, newFieldTy, fieldAddr);
} else {
initStackCopy(fieldTI, fieldTy, fieldAddr);
}
break;
}
case ParameterConvention::Indirect_In_Guaranteed:
if (outType->isNoEscape()) {
cast<LoadableTypeInfo>(fieldTI).loadAsCopy(subIGF, fieldAddr, param);
} else {
// The argument is +0, so we can use the address of the param in
// the context directly.
param.add(fieldAddr.getAddress());
dependsOnContextLifetime = true;
}
break;
case ParameterConvention::Indirect_Inout:
case ParameterConvention::Indirect_InoutAliasable:
// Load the address of the inout parameter.
cast<LoadableTypeInfo>(fieldTI).loadAsCopy(subIGF, fieldAddr, param);
break;
case ParameterConvention::Direct_Guaranteed:
case ParameterConvention::Direct_Unowned:
// If the type is nontrivial, keep the context alive since the field
// depends on the context to not be deallocated.
if (!fieldTI.isPOD(ResilienceExpansion::Maximal))
dependsOnContextLifetime = true;
// Load these parameters directly. We can "take" since the parameter is
// +0. This can happen since the context will keep the parameter alive.
cast<LoadableTypeInfo>(fieldTI).loadAsTake(subIGF, fieldAddr, param);
break;
case ParameterConvention::Direct_Owned:
// Copy the value out at +1.
cast<LoadableTypeInfo>(fieldTI).loadAsCopy(subIGF, fieldAddr, param);
break;
}
// Reemit the capture params as unsubstituted.
// Skip empty parameters.
while (origParamI < origType->getParameters().size()) {
if (!isABIIgnoredParameterWithoutStorage(IGM, subIGF, substType,
origParamI))
break;
origParamI++;
}
if (origParamI < origType->getParameters().size()) {
Explosion origParam;
auto origParamInfo = origType->getParameters()[origParamI];
if (hasPolymorphicParams)
bindPolymorphicParameter(subIGF, origType, substType, param,
origParamI);
emitApplyArgument(subIGF,
origType, origParamInfo,
substType, substType->getParameters()[origParamI],
param, origParam);
bool isWitnessMethodCalleeSelf = (isWitnessMethodCallee &&
origParamI + 1 == origType->getParameters().size());
needsAllocas |= addNativeArgument(
subIGF, origParam, origType, origParamInfo,
isWitnessMethodCalleeSelf ? witnessMethodSelfValue : args, false);
++origParamI;
} else {
args.add(param.claimAll());
}
}
// If the parameters can live independent of the context, release it now
// so we can tail call. The safety of this assumes that neither this release
// nor any of the loads can throw.
if (consumesContext && !dependsOnContextLifetime && rawData) {
assert(!outType->isNoEscape() && "Trivial context must not be released");
subIGF.emitNativeStrongRelease(rawData, subIGF.getDefaultAtomicity());
}
// Now that we have bound generic parameters from the captured arguments
// emit the polymorphic arguments.
if (hasPolymorphicParameters(origType)) {
emitPolymorphicArguments(subIGF, origType, subs,
&witnessMetadata, polyArgs);
}
}
// Derive the callee function pointer.
auto fnTy = origSig.getType()->getPointerTo();
FunctionPointer fnPtr = [&]() -> FunctionPointer {
// If we found a function pointer statically, great.
if (staticFnPtr) {
if (staticFnPtr->getPointer()->getType() != fnTy) {
auto fnPtr = staticFnPtr->getPointer();
fnPtr = subIGF.Builder.CreateBitCast(fnPtr, fnTy);
return FunctionPointer(fnPtr, origSig);
}
return *staticFnPtr;
}
// Otherwise, it was the last thing we added to the layout.
// The dynamic function pointer is packed "last" into the context,
// and we pulled it out as an argument. Just pop it off.
auto fnPtr = args.takeLast();
// It comes out of the context as an i8*. Cast to the function type.
fnPtr = subIGF.Builder.CreateBitCast(fnPtr, fnTy);
assert(lastCapturedFieldPtr);
auto authInfo = PointerAuthInfo::emit(subIGF,
IGM.getOptions().PointerAuth.PartialApplyCapture,
lastCapturedFieldPtr,
PointerAuthEntity::Special::PartialApplyCapture);
return FunctionPointer(fnPtr, authInfo, origSig);
}();
// Derive the context argument if needed. This is either:
// - the saved context argument, in which case it was the last
// thing we added to the layout other than a possible non-static
// function pointer (which we already popped off of 'args'); or
// - 'self', in which case it was the last formal argument.
// In either case, it's the last thing in 'args'.
llvm::Value *fnContext = nullptr;
if (haveContextArgument)
fnContext = args.takeLast();
polyArgs.transferInto(args, polyArgs.size());
// If we have a witness method call, the inner context is the
// witness table. Metadata for Self is derived inside the partial
// application thunk and doesn't need to be stored in the outer
// context.
if (isWitnessMethodCallee) {
assert(fnContext->getType() == IGM.Int8PtrTy);
llvm::Value *wtable = subIGF.Builder.CreateBitCast(
fnContext, IGM.WitnessTablePtrTy);
assert(wtable->getType() == IGM.WitnessTablePtrTy);
witnessMetadata.SelfWitnessTable = wtable;
// Okay, this is where the callee context goes.
} else if (fnContext) {
args.add(fnContext);
// Pass a placeholder for thin function calls.
} else if (origType->hasErrorResult()) {
args.add(llvm::UndefValue::get(IGM.RefCountedPtrTy));
}
// Add the witness methods self argument before the error parameter after the
// polymorphic arguments.
if (isWitnessMethodCallee)
witnessMethodSelfValue.transferInto(args, witnessMethodSelfValue.size());
// Pass down the error result.
if (origType->hasErrorResult()) {
llvm::Value *errorResultPtr = origParams.claimNext();
args.add(errorResultPtr);
}
assert(origParams.empty());
if (isWitnessMethodCallee) {
assert(witnessMetadata.SelfMetadata->getType() == IGM.TypeMetadataPtrTy);
args.add(witnessMetadata.SelfMetadata);
assert(witnessMetadata.SelfWitnessTable->getType() == IGM.WitnessTablePtrTy);
args.add(witnessMetadata.SelfWitnessTable);
}
llvm::CallInst *call = subIGF.Builder.CreateCall(fnPtr, args.claimAll());
if (addressesToDeallocate.empty() && !needsAllocas &&
(!consumesContext || !dependsOnContextLifetime))
call->setTailCall();
// Deallocate everything we allocated above.
// FIXME: exceptions?
for (auto &entry : addressesToDeallocate) {
entry.TI.deallocateStack(subIGF, entry.Addr, entry.Type);
}
// If the parameters depended on the context, consume the context now.
if (rawData && consumesContext && dependsOnContextLifetime) {
assert(!outType->isNoEscape() && "Trivial context must not be released");
subIGF.emitNativeStrongRelease(rawData, subIGF.getDefaultAtomicity());
}
// Reabstract the result value as substituted.
SILFunctionConventions origConv(origType, IGM.getSILModule());
auto &outResultTI = IGM.getTypeInfo(outConv.getSILResultType());
auto &nativeResultSchema = outResultTI.nativeReturnValueSchema(IGM);
if (call->getType()->isVoidTy()) {
if (!resultValueAddr.isValid())
subIGF.Builder.CreateRetVoid();
else {
// Okay, we have called a function that expects an indirect return type
// but the partially applied return type is direct.
assert(!nativeResultSchema.requiresIndirect());
Explosion loadedResult;
cast<LoadableTypeInfo>(outResultTI)
.loadAsTake(subIGF, resultValueAddr, loadedResult);
Explosion nativeResult = nativeResultSchema.mapIntoNative(
IGM, subIGF, loadedResult, outConv.getSILResultType(), false);
outResultTI.deallocateStack(subIGF, resultValueAddr,
outConv.getSILResultType());
if (nativeResult.size() == 1)
subIGF.Builder.CreateRet(nativeResult.claimNext());
else {
llvm::Value *nativeAgg =
llvm::UndefValue::get(nativeResultSchema.getExpandedType(IGM));
for (unsigned i = 0, e = nativeResult.size(); i != e; ++i) {
auto *elt = nativeResult.claimNext();
nativeAgg = subIGF.Builder.CreateInsertValue(nativeAgg, elt, i);
}
subIGF.Builder.CreateRet(nativeAgg);
}
}
} else {
llvm::Value *callResult = call;
// If the result type is dependent on a type parameter we might have to
// cast to the result type - it could be substituted.
if (origConv.getSILResultType().hasTypeParameter()) {
auto ResType = fwd->getReturnType();
if (ResType != callResult->getType())
callResult = subIGF.coerceValue(callResult, ResType, subIGF.IGM.DataLayout);
}
subIGF.Builder.CreateRet(callResult);
}
return fwd;
}
/// Emit a partial application thunk for a function pointer applied to a partial
/// set of argument values.
Optional<StackAddress> irgen::emitFunctionPartialApplication(
IRGenFunction &IGF, SILFunction &SILFn, const FunctionPointer &fn,
llvm::Value *fnContext, Explosion &args, ArrayRef<SILParameterInfo> params,
SubstitutionMap subs, CanSILFunctionType origType,
CanSILFunctionType substType, CanSILFunctionType outType, Explosion &out,
bool isOutlined) {
// If we have a single Swift-refcounted context value, we can adopt it
// directly as our closure context without creating a box and thunk.
enum HasSingleSwiftRefcountedContext { Maybe, Yes, No, Thunkable }
hasSingleSwiftRefcountedContext = Maybe;
Optional<ParameterConvention> singleRefcountedConvention;
SmallVector<const TypeInfo *, 4> argTypeInfos;
SmallVector<SILType, 4> argValTypes;
SmallVector<ParameterConvention, 4> argConventions;
// A context's HeapLayout stores all of the partially applied args.
// A HeapLayout is "fixed" if all of its fields have a fixed layout.
// Otherwise the HeapLayout is "non-fixed".
// Only a non-fixed HeapLayout needs TypeMetadata of the non-fixed fields
// during IRGen of the HeapLayout's destructor function.
// We should not consider partially applied args as TypeMetadata sources,
// because they are available only in the caller and the partial application
// forwarder, but not in the destructor function.
// It is safe to consider partially applied args as TypeMetadata sources for
// "fixed" HeapLayout, because they are not accessed during the IRGen of the
// destructor function.
bool considerParameterSources = true;
for (auto param : params) {
SILType argType = IGF.IGM.silConv.getSILType(param, origType);
auto argLoweringTy = getArgumentLoweringType(argType.getASTType(), param,
outType->isNoEscape());
auto &ti = IGF.getTypeInfoForLowered(argLoweringTy);
if (!isa<FixedTypeInfo>(ti)) {
considerParameterSources = false;
break;
}
}
// Reserve space for polymorphic bindings.
auto bindings = NecessaryBindings::forPartialApplyForwarder(
IGF.IGM, origType, subs, considerParameterSources);
if (!bindings.empty()) {
hasSingleSwiftRefcountedContext = No;
auto bindingsSize = bindings.getBufferSize(IGF.IGM);
auto &bindingsTI = IGF.IGM.getOpaqueStorageTypeInfo(bindingsSize,
IGF.IGM.getPointerAlignment());
argValTypes.push_back(SILType());
argTypeInfos.push_back(&bindingsTI);
argConventions.push_back(ParameterConvention::Direct_Unowned);
}
// Collect the type infos for the context parameters.
for (auto param : params) {
SILType argType = IGF.IGM.silConv.getSILType(param, origType);
auto argLoweringTy = getArgumentLoweringType(argType.getASTType(), param,
outType->isNoEscape());
auto &ti = IGF.getTypeInfoForLowered(argLoweringTy);
// Empty values don't matter.
auto schema = ti.getSchema();
if (schema.empty() && !param.isFormalIndirect())
continue;
argValTypes.push_back(argType);
argConventions.push_back(param.getConvention());
argTypeInfos.push_back(&ti);
// Update the single-swift-refcounted check, unless we already ruled that
// out.
if (hasSingleSwiftRefcountedContext == No)
continue;
// Adding nonempty values when we already have a single refcounted pointer
// means we don't have a single value anymore.
if (hasSingleSwiftRefcountedContext != Maybe) {
hasSingleSwiftRefcountedContext = No;
continue;
}
if (ti.isSingleSwiftRetainablePointer(ResilienceExpansion::Maximal)) {
hasSingleSwiftRefcountedContext = Yes;
singleRefcountedConvention = param.getConvention();
} else {
hasSingleSwiftRefcountedContext = No;
}
}
// We can't just bitcast if there's an error parameter to forward.
// This is an unfortunate restriction arising from the fact that a
// thin throwing function will have the signature:
// %result (%arg*, %context*, %error*)
// but the output signature needs to be
// %result (%context*, %error*)
//
// 'swifterror' fixes this physically, but there's still a risk of
// miscompiles because the LLVM optimizer may forward arguments
// positionally without considering 'swifterror'.
//
// Note, however, that we will override this decision below if the
// only thing we have to forward is already a context pointer.
// That's fine.
//
// The proper long-term fix is that closure functions should be
// emitted with a convention that takes the closure box as the
// context parameter. When we do that, all of this code will
// disappear.
if (hasSingleSwiftRefcountedContext == Yes &&
origType->hasErrorResult()) {
hasSingleSwiftRefcountedContext = Thunkable;
}
// If the function pointer is a witness method call, include the witness
// table in the context.
if (origType->getRepresentation() ==
SILFunctionTypeRepresentation::WitnessMethod) {
llvm::Value *wtable = fnContext;
assert(wtable->getType() == IGF.IGM.WitnessTablePtrTy);
// TheRawPointerType lowers as i8*, not i8**.
args.add(IGF.Builder.CreateBitCast(wtable, IGF.IGM.Int8PtrTy));
argValTypes.push_back(SILType::getRawPointerType(IGF.IGM.Context));
argTypeInfos.push_back(
&IGF.getTypeInfoForLowered(IGF.IGM.Context.TheRawPointerType));
argConventions.push_back(ParameterConvention::Direct_Unowned);
hasSingleSwiftRefcountedContext = No;
// Otherwise, we might have a reference-counted context pointer.
} else if (fnContext) {
args.add(fnContext);
argValTypes.push_back(SILType::getNativeObjectType(IGF.IGM.Context));
argConventions.push_back(origType->getCalleeConvention());
argTypeInfos.push_back(
&IGF.getTypeInfoForLowered(IGF.IGM.Context.TheNativeObjectType));
// If this is the only context argument we end up with, we can just share
// it.
if (args.size() == 1) {
assert(bindings.empty());
hasSingleSwiftRefcountedContext = Yes;
singleRefcountedConvention = origType->getCalleeConvention();
}
}
auto outAuthInfo = PointerAuthInfo::forFunctionPointer(IGF.IGM, outType);
// If we have a single refcounted pointer context (and no polymorphic args
// to capture), and the dest ownership semantics match the parameter's,
// skip building the box and thunk and just take the pointer as
// context.
// TODO: We can only do this and use swiftself if all our swiftcc emit the
// last parameter that fits into a register as swiftself.
// We should get this optimization back using the @convention(closure) whose
// box argument should just be swift self.
if (/* DISABLES CODE */ (false) &&
!origType->isPolymorphic() &&
hasSingleSwiftRefcountedContext == Yes &&
outType->getCalleeConvention() == *singleRefcountedConvention) {
assert(args.size() == 1);
auto fnPtr = emitPointerAuthResign(IGF, fn, outAuthInfo).getPointer();
fnPtr = IGF.Builder.CreateBitCast(fnPtr, IGF.IGM.Int8PtrTy);
out.add(fnPtr);
llvm::Value *ctx = args.claimNext();
ctx = IGF.Builder.CreateBitCast(ctx, IGF.IGM.RefCountedPtrTy);
out.add(ctx);
return {};
}
Optional<FunctionPointer> staticFn;
if (fn.isConstant()) staticFn = fn;
// If the function pointer is dynamic, include it in the context.
size_t nonStaticFnIndex = ~size_t(0);
if (!staticFn) {
nonStaticFnIndex = argTypeInfos.size();
argValTypes.push_back(SILType::getRawPointerType(IGF.IGM.Context));
argTypeInfos.push_back(
&IGF.getTypeInfoForLowered(IGF.IGM.Context.TheRawPointerType));
argConventions.push_back(ParameterConvention::Direct_Unowned);
hasSingleSwiftRefcountedContext = No;
}
// If we only need to capture a single Swift-refcounted object, we
// still need to build a thunk, but we don't need to allocate anything.
if ((hasSingleSwiftRefcountedContext == Yes ||
hasSingleSwiftRefcountedContext == Thunkable) &&
*singleRefcountedConvention != ParameterConvention::Indirect_Inout &&
*singleRefcountedConvention !=
ParameterConvention::Indirect_InoutAliasable) {
assert(bindings.empty());
assert(args.size() == 1);
auto origSig = IGF.IGM.getSignature(origType);
llvm::Value *forwarder =
emitPartialApplicationForwarder(IGF.IGM, staticFn, fnContext != nullptr,
origSig, origType, substType,
outType, subs, nullptr, argConventions);
forwarder = emitPointerAuthSign(IGF, forwarder, outAuthInfo);
forwarder = IGF.Builder.CreateBitCast(forwarder, IGF.IGM.Int8PtrTy);
out.add(forwarder);
llvm::Value *ctx = args.claimNext();
if (isIndirectFormalParameter(*singleRefcountedConvention))
ctx = IGF.Builder.CreateLoad(ctx, IGF.IGM.getPointerAlignment());
auto expectedClosureTy =
outType->isNoEscape() ? IGF.IGM.OpaquePtrTy : IGF.IGM.RefCountedPtrTy;
// We might get a struct containing a pointer e.g type <{ %AClass* }>
if (ctx->getType() != expectedClosureTy)
ctx = IGF.coerceValue(ctx, expectedClosureTy, IGF.IGM.DataLayout);
out.add(ctx);
if (outType->isNoEscape())
return StackAddress();
return {};
}
// Store the context arguments on the heap/stack.
assert(argValTypes.size() == argTypeInfos.size()
&& argTypeInfos.size() == argConventions.size()
&& "argument info lists out of sync");
HeapLayout layout(IGF.IGM, LayoutStrategy::Optimal, argValTypes, argTypeInfos,
/*typeToFill*/ nullptr,
std::move(bindings));
llvm::Value *data;
Optional<StackAddress> stackAddr;
if (args.empty() && layout.isKnownEmpty()) {
if (outType->isNoEscape())
data = llvm::ConstantPointerNull::get(IGF.IGM.OpaquePtrTy);
else
data = IGF.IGM.RefCountedNull;
} else {
// Allocate a new object on the heap or stack.
HeapNonFixedOffsets offsets(IGF, layout);
if (outType->isNoEscape()) {
stackAddr = IGF.emitDynamicAlloca(
IGF.IGM.Int8Ty, layout.isFixedLayout() ? layout.emitSize(IGF.IGM) : offsets.getSize() , Alignment(16));
stackAddr = stackAddr->withAddress(IGF.Builder.CreateBitCast(
stackAddr->getAddress(), IGF.IGM.OpaquePtrTy));
data = stackAddr->getAddress().getAddress();
} else {
auto descriptor = IGF.IGM.getAddrOfCaptureDescriptor(SILFn, origType,
substType, subs,
layout);
data = IGF.emitUnmanagedAlloc(layout, "closure", descriptor, &offsets);
}
Address dataAddr = layout.emitCastTo(IGF, data);
unsigned i = 0;
// Store necessary bindings, if we have them.
if (layout.hasBindings()) {
auto &bindingsLayout = layout.getElement(i);
Address bindingsAddr = bindingsLayout.project(IGF, dataAddr, offsets);
layout.getBindings().save(IGF, bindingsAddr);
++i;
}
// Store the context arguments.
for (unsigned end = layout.getElements().size(); i < end; ++i) {
auto &fieldLayout = layout.getElement(i);
auto &fieldTy = layout.getElementTypes()[i];
Address fieldAddr = fieldLayout.project(IGF, dataAddr, offsets);
// We don't add non-constant function pointers to the explosion above,
// so we need to handle them specially now.
if (i == nonStaticFnIndex) {
llvm::Value *fnPtr;
if (auto &schema = IGF.getOptions().PointerAuth.PartialApplyCapture) {
auto schemaAuthInfo =
PointerAuthInfo::emit(IGF, schema, fieldAddr.getAddress(),
PointerAuthEntity::Special::PartialApplyCapture);
fnPtr = emitPointerAuthResign(IGF, fn, schemaAuthInfo).getPointer();
} else {
fnPtr = fn.getPointer();
}
fnPtr = IGF.Builder.CreateBitCast(fnPtr, IGF.IGM.Int8PtrTy);
IGF.Builder.CreateStore(fnPtr, fieldAddr);
continue;
}
switch (argConventions[i]) {
// Take indirect value arguments out of memory.
case ParameterConvention::Indirect_In:
case ParameterConvention::Indirect_In_Constant:
case ParameterConvention::Indirect_In_Guaranteed: {
if (outType->isNoEscape()) {
cast<LoadableTypeInfo>(fieldLayout.getType())
.initialize(IGF, args, fieldAddr, isOutlined);
} else {
auto addr =
fieldLayout.getType().getAddressForPointer(args.claimNext());
fieldLayout.getType().initializeWithTake(IGF, fieldAddr, addr,
fieldTy, isOutlined);
}
break;
}
// Take direct value arguments and inout pointers by value.
case ParameterConvention::Direct_Unowned:
case ParameterConvention::Direct_Owned:
case ParameterConvention::Direct_Guaranteed:
case ParameterConvention::Indirect_Inout:
case ParameterConvention::Indirect_InoutAliasable:
cast<LoadableTypeInfo>(fieldLayout.getType())
.initialize(IGF, args, fieldAddr, isOutlined);
break;
}
}
}
assert(args.empty() && "unused args in partial application?!");
// Create the forwarding stub.
auto origSig = IGF.IGM.getSignature(origType);
llvm::Value *forwarder = emitPartialApplicationForwarder(IGF.IGM,
staticFn,
fnContext != nullptr,
origSig,
origType,
substType,
outType,
subs,
&layout,
argConventions);
forwarder = emitPointerAuthSign(IGF, forwarder, outAuthInfo);
forwarder = IGF.Builder.CreateBitCast(forwarder, IGF.IGM.Int8PtrTy);
out.add(forwarder);
out.add(data);
return stackAddr;
}
/// Emit the block copy helper for a block.
static llvm::Function *emitBlockCopyHelper(IRGenModule &IGM,
CanSILBlockStorageType blockTy,
const BlockStorageTypeInfo &blockTL){
// See if we've produced a block copy helper for this type before.
// TODO
// Create the helper.
llvm::Type *args[] = {
blockTL.getStorageType()->getPointerTo(),
blockTL.getStorageType()->getPointerTo(),
};
auto copyTy = llvm::FunctionType::get(IGM.VoidTy, args, /*vararg*/ false);
// TODO: Give these predictable mangled names and shared linkage.
auto func = llvm::Function::Create(copyTy, llvm::GlobalValue::InternalLinkage,
"block_copy_helper",
IGM.getModule());
func->setAttributes(IGM.constructInitialAttributes());
IRGenFunction IGF(IGM, func);
if (IGM.DebugInfo)
IGM.DebugInfo->emitArtificialFunction(IGF, func);
// Copy the captures from the source to the destination.
Explosion params = IGF.collectParameters();
auto dest = Address(params.claimNext(), blockTL.getFixedAlignment());
auto src = Address(params.claimNext(), blockTL.getFixedAlignment());
auto destCapture = blockTL.projectCapture(IGF, dest);
auto srcCapture = blockTL.projectCapture(IGF, src);
auto &captureTL = IGM.getTypeInfoForLowered(blockTy->getCaptureType());
captureTL.initializeWithCopy(IGF, destCapture, srcCapture,
blockTy->getCaptureAddressType(), false);
IGF.Builder.CreateRetVoid();
return func;
}
/// Emit the block copy helper for a block.
static llvm::Function *emitBlockDisposeHelper(IRGenModule &IGM,
CanSILBlockStorageType blockTy,
const BlockStorageTypeInfo &blockTL){
// See if we've produced a block destroy helper for this type before.
// TODO
// Create the helper.
auto destroyTy = llvm::FunctionType::get(IGM.VoidTy,
blockTL.getStorageType()->getPointerTo(),
/*vararg*/ false);
// TODO: Give these predictable mangled names and shared linkage.
auto func = llvm::Function::Create(destroyTy,
llvm::GlobalValue::InternalLinkage,
"block_destroy_helper",
IGM.getModule());
func->setAttributes(IGM.constructInitialAttributes());
IRGenFunction IGF(IGM, func);
assert(!func->hasFnAttribute(llvm::Attribute::SanitizeThread));
if (IGM.DebugInfo)
IGM.DebugInfo->emitArtificialFunction(IGF, func);
// Destroy the captures.
Explosion params = IGF.collectParameters();
auto storage = Address(params.claimNext(), blockTL.getFixedAlignment());
auto capture = blockTL.projectCapture(IGF, storage);
auto &captureTL = IGM.getTypeInfoForLowered(blockTy->getCaptureType());
captureTL.destroy(IGF, capture, blockTy->getCaptureAddressType(),
false /*block storage code path: never outlined*/);
IGF.Builder.CreateRetVoid();
return func;
}
/// Emit the block header into a block storage slot.
void irgen::emitBlockHeader(IRGenFunction &IGF,
Address storage,
CanSILBlockStorageType blockTy,
llvm::Constant *invokeFunction,
CanSILFunctionType invokeTy,
ForeignFunctionInfo foreignInfo) {
auto &storageTL
= IGF.getTypeInfoForLowered(blockTy).as<BlockStorageTypeInfo>();
Address headerAddr = storageTL.projectBlockHeader(IGF, storage);
//
// Initialize the "isa" pointer, which is _NSConcreteStackBlock.
auto NSConcreteStackBlock =
IGF.IGM.getModule()->getOrInsertGlobal("_NSConcreteStackBlock",
IGF.IGM.ObjCClassStructTy);
ApplyIRLinkage(IRLinkage::ExternalImport)
.to(cast<llvm::GlobalVariable>(NSConcreteStackBlock));
//
// Set the flags.
// - HAS_COPY_DISPOSE unless the capture type is POD
uint32_t flags = 0;
auto &captureTL
= IGF.getTypeInfoForLowered(blockTy->getCaptureType());
bool isPOD = captureTL.isPOD(ResilienceExpansion::Maximal);
if (!isPOD)
flags |= 1 << 25;
// - HAS_STRET, if the invoke function is sret
assert(foreignInfo.ClangInfo);
if (foreignInfo.ClangInfo->getReturnInfo().isIndirect())
flags |= 1 << 29;
// - HAS_SIGNATURE
flags |= 1 << 30;
auto flagsVal = llvm::ConstantInt::get(IGF.IGM.Int32Ty, flags);
// Collect the reserved and invoke pointer fields.
auto reserved = llvm::ConstantInt::get(IGF.IGM.Int32Ty, 0);
llvm::Value *invokeVal = llvm::ConstantExpr::getBitCast(invokeFunction,
IGF.IGM.FunctionPtrTy);
// Build the block descriptor.
ConstantInitBuilder builder(IGF.IGM);
auto descriptorFields = builder.beginStruct();
const clang::ASTContext &ASTContext = IGF.IGM.getClangASTContext();
llvm::IntegerType *UnsignedLongTy =
llvm::IntegerType::get(IGF.IGM.LLVMContext,
ASTContext.getTypeSize(ASTContext.UnsignedLongTy));
descriptorFields.addInt(UnsignedLongTy, 0);
descriptorFields.addInt(UnsignedLongTy,
storageTL.getFixedSize().getValue());
if (!isPOD) {
// Define the copy and dispose helpers.
descriptorFields.addSignedPointer(
emitBlockCopyHelper(IGF.IGM, blockTy, storageTL),
IGF.getOptions().PointerAuth.BlockHelperFunctionPointers,
PointerAuthEntity::Special::BlockCopyHelper);
descriptorFields.addSignedPointer(
emitBlockDisposeHelper(IGF.IGM, blockTy, storageTL),
IGF.getOptions().PointerAuth.BlockHelperFunctionPointers,
PointerAuthEntity::Special::BlockDisposeHelper);
}
// Build the descriptor signature.
descriptorFields.add(getBlockTypeExtendedEncoding(IGF.IGM, invokeTy));
// Create the descriptor.
auto descriptor =
descriptorFields.finishAndCreateGlobal("block_descriptor",
IGF.IGM.getPointerAlignment(),
/*constant*/ true);
auto descriptorVal = llvm::ConstantExpr::getBitCast(descriptor,
IGF.IGM.Int8PtrTy);
// Store the block header.
auto layout = IGF.IGM.DataLayout.getStructLayout(IGF.IGM.ObjCBlockStructTy);
IGF.Builder.CreateStore(NSConcreteStackBlock,
IGF.Builder.CreateStructGEP(headerAddr, 0, layout));
IGF.Builder.CreateStore(flagsVal,
IGF.Builder.CreateStructGEP(headerAddr, 1, layout));
IGF.Builder.CreateStore(reserved,
IGF.Builder.CreateStructGEP(headerAddr, 2, layout));
auto invokeAddr = IGF.Builder.CreateStructGEP(headerAddr, 3, layout);
if (auto &schema =
IGF.getOptions().PointerAuth.BlockInvocationFunctionPointers) {
auto invokeAuthInfo = PointerAuthInfo::emit(IGF, schema,
invokeAddr.getAddress(),
invokeTy);
invokeVal = emitPointerAuthSign(IGF, invokeVal, invokeAuthInfo);
}
IGF.Builder.CreateStore(invokeVal, invokeAddr);
IGF.Builder.CreateStore(descriptorVal,
IGF.Builder.CreateStructGEP(headerAddr, 4, layout));
}
Reference in New Issue View Git Blame Copy Permalink