averello/swift-mirror
1
0
Fork 0
mirror of https://github.com/apple/swift.git synced 2025-12-21 12:14:44 +01:00
Code Issues Packages Projects Releases Wiki Activity

[NFC] Clean up some of the code in the loadable-by-address pass

Browse Source
This commit is contained in:
John McCall
2018-11-05 17:44:06 -05:00
parent 5fc36459ec
commit fde1ce759b
1 changed files with 221 additions and 266 deletions
Download Patch File Download Diff File Expand all files Collapse all files

487
lib/IRGen/LoadableByAddress.cpp
View File

@@ -529,6 +529,14 @@ struct StructLoweringState {
StructLoweringState(SILFunction *F, irgen::IRGenModule &Mod,
LargeSILTypeMapper &Mapper)
: F(F), Mod(Mod), Mapper(Mapper) {}
bool isLargeLoadableType(SILType type) {
return ::isLargeLoadableType(F->getGenericEnvironment(), type, Mod);
}
SILType getNewSILType(SILType type) {
return Mapper.getNewSILType(F->getGenericEnvironment(), type, Mod);
}
};
} // end anonymous namespace
@@ -695,11 +703,10 @@ void LargeValueVisitor::visitApply(ApplySite applySite) {
if (!modifiableApply(applySite, pass.Mod)) {
return visitInstr(applySite.getInstruction());
}
GenericEnvironment *genEnv = pass.F->getGenericEnvironment();
for (Operand &operand : applySite.getArgumentOperands()) {
SILValue currOperand = operand.get();
SILType silType = currOperand->getType();
SILType newSilType = pass.Mapper.getNewSILType(genEnv, silType, pass.Mod);
SILType newSilType = pass.getNewSILType(silType);
if (silType != newSilType ||
std::find(pass.largeLoadableArgs.begin(), pass.largeLoadableArgs.end(),
currOperand) != pass.largeLoadableArgs.end() ||
@@ -715,8 +722,7 @@ void LargeValueVisitor::visitApply(ApplySite applySite) {
if (auto beginApply = dyn_cast<BeginApplyInst>(applySite)) {
for (auto yield : beginApply->getYieldedValues()) {
auto oldYieldType = yield->getType();
auto newYieldType =
pass.Mapper.getNewSILType(genEnv, oldYieldType, pass.Mod);
auto newYieldType = pass.getNewSILType(oldYieldType);
if (oldYieldType != newYieldType) {
pass.applies.push_back(applySite.getInstruction());
return;
@@ -726,9 +732,9 @@ void LargeValueVisitor::visitApply(ApplySite applySite) {
}
SILType currType = applySite.getType();
SILType newType = pass.Mapper.getNewSILType(genEnv, currType, pass.Mod);
SILType newType = pass.getNewSILType(currType);
// We only care about function type results
if (!isLargeLoadableType(genEnv, currType, pass.Mod) &&
if (!pass.isLargeLoadableType(currType) &&
(currType != newType)) {
pass.applies.push_back(applySite.getInstruction());
return;
@@ -822,9 +828,7 @@ void LargeValueVisitor::visitSwitchEnumInst(SwitchEnumInst *instr) {
for (SILArgument *arg : currBB->getArguments()) {
if (pass.Mapper.shouldConvertBBArg(arg, pass.Mod)) {
SILType storageType = arg->getType();
auto *genEnv = instr->getFunction()->getGenericEnvironment();
SILType newSILType =
pass.Mapper.getNewSILType(genEnv, storageType, pass.Mod);
SILType newSILType = pass.getNewSILType(storageType);
if (newSILType.isAddress()) {
pass.switchEnumInstsToMod.push_back(instr);
return;
@@ -881,9 +885,8 @@ void LargeValueVisitor::visitDestroyValueInst(DestroyValueInst *instr) {
}
void LargeValueVisitor::visitResultTyInst(SingleValueInstruction *instr) {
GenericEnvironment *genEnv = instr->getFunction()->getGenericEnvironment();
SILType currSILType = instr->getType().getObjectType();
SILType newSILType = pass.Mapper.getNewSILType(genEnv, currSILType, pass.Mod);
SILType newSILType = pass.getNewSILType(currSILType);
if (currSILType != newSILType) {
pass.resultTyInstsToMod.insert(instr);
}
@@ -976,16 +979,17 @@ void LargeValueVisitor::visitInstr(SILInstruction *instr) {
namespace {
class LoadableStorageAllocation {
public:
StructLoweringState &pass;
public:
explicit LoadableStorageAllocation(StructLoweringState &pass) : pass(pass) {}
void allocateLoadableStorage();
void replaceLoadWithCopyAddr(LoadInst *optimizableLoad);
void replaceLoadWithCopyAddrForModifiable(LoadInst *unoptimizableLoad);
void replaceLoad(LoadInst *unoptimizableLoad);
protected:
void replaceLoadWithCopyAddr(LoadInst *optimizableLoad);
void replaceLoadWithCopyAddrForModifiable(LoadInst *unoptimizableLoad);
void convertIndirectFunctionArgs();
void insertIndirectReturnArgs();
void convertIndirectFunctionPointerArgsForUnmodifiable();
@@ -998,6 +1002,43 @@ protected:
};
} // end anonymous namespace
static AllocStackInst *allocate(StructLoweringState &pass,
SILLocation loc, SILType type) {
assert(type.isObject());
// Insert an alloc_stack at the beginning of the function.
SILBuilderWithScope allocBuilder(&*pass.F->begin());
AllocStackInst *alloc = allocBuilder.createAllocStack(loc, type);
// Insert dealloc_stack at the end(s) of the function.
for (TermInst *termInst : pass.returnInsts) {
SILBuilderWithScope deallocBuilder(termInst);
deallocBuilder.createDeallocStack(loc, alloc);
}
return alloc;
}
static StoreOwnershipQualifier
getStoreInitOwnership(StructLoweringState &pass, SILType type) {
if (!pass.F->hasQualifiedOwnership()) {
return StoreOwnershipQualifier::Unqualified;
} else if (type.isTrivial(pass.F->getModule())) {
return StoreOwnershipQualifier::Trivial;
} else {
return StoreOwnershipQualifier::Init;
}
}
static StoreInst *createStoreInit(StructLoweringState &pass,
SILBasicBlock::iterator where,
SILLocation loc,
SILValue value, SILValue address) {
SILBuilderWithScope storeBuilder(where);
return storeBuilder.createStore(loc, value, address,
getStoreInitOwnership(pass, value->getType()));
}
static SILInstruction *createOutlinedCopyCall(SILBuilder &copyBuilder,
SILValue src, SILValue tgt,
StructLoweringState &pass,
@@ -1012,19 +1053,12 @@ void LoadableStorageAllocation::replaceLoadWithCopyAddr(
LoadInst *optimizableLoad) {
SILValue value = optimizableLoad->getOperand();
SILBuilderWithScope allocBuilder(&*pass.F->begin());
AllocStackInst *allocInstr =
allocBuilder.createAllocStack(value.getLoc(), value->getType());
auto allocInstr = allocate(pass, value.getLoc(),
value->getType().getObjectType());
SILBuilderWithScope outlinedBuilder(optimizableLoad);
createOutlinedCopyCall(outlinedBuilder, value, allocInstr, pass);
// Insert stack deallocations.
for (TermInst *termInst : pass.returnInsts) {
SILBuilderWithScope deallocBuilder(termInst);
deallocBuilder.createDeallocStack(allocInstr->getLoc(), allocInstr);
}
for (auto *user : optimizableLoad->getUses()) {
SILInstruction *userIns = user->getUser();
switch (userIns->getKind()) {
@@ -1042,37 +1076,32 @@ void LoadableStorageAllocation::replaceLoadWithCopyAddr(
break;
}
case SILInstructionKind::RetainValueInst: {
auto *insToInsert = dyn_cast<RetainValueInst>(userIns);
assert(insToInsert && "Unexpected cast failure");
auto *insToInsert = cast<RetainValueInst>(userIns);
pass.retainInstsToMod.push_back(insToInsert);
break;
}
case SILInstructionKind::ReleaseValueInst: {
auto *insToInsert = dyn_cast<ReleaseValueInst>(userIns);
assert(insToInsert && "Unexpected cast failure");
auto *insToInsert = cast<ReleaseValueInst>(userIns);
pass.releaseInstsToMod.push_back(insToInsert);
break;
}
case SILInstructionKind::StoreInst: {
auto *insToInsert = dyn_cast<StoreInst>(userIns);
assert(insToInsert && "Unexpected cast failure");
auto *insToInsert = cast<StoreInst>(userIns);
pass.storeInstsToMod.push_back(insToInsert);
break;
}
case SILInstructionKind::DebugValueInst: {
auto *insToInsert = dyn_cast<DebugValueInst>(userIns);
assert(insToInsert && "Unexpected cast failure");
auto *insToInsert = cast<DebugValueInst>(userIns);
pass.debugInstsToMod.push_back(insToInsert);
break;
}
case SILInstructionKind::DestroyValueInst: {
auto *insToInsert = dyn_cast<DestroyValueInst>(userIns);
assert(insToInsert && "Unexpected cast failure");
auto *insToInsert = cast<DestroyValueInst>(userIns);
pass.destroyValueInstsToMod.push_back(insToInsert);
break;
}
case SILInstructionKind::StructExtractInst: {
auto *instToInsert = dyn_cast<StructExtractInst>(userIns);
auto *instToInsert = cast<StructExtractInst>(userIns);
if (std::find(pass.structExtractInstsToMod.begin(),
pass.structExtractInstsToMod.end(),
instToInsert) == pass.structExtractInstsToMod.end()) {
@@ -1081,7 +1110,7 @@ void LoadableStorageAllocation::replaceLoadWithCopyAddr(
break;
}
case SILInstructionKind::SwitchEnumInst: {
auto *instToInsert = dyn_cast<SwitchEnumInst>(userIns);
auto *instToInsert = cast<SwitchEnumInst>(userIns);
if (std::find(pass.switchEnumInstsToMod.begin(),
pass.switchEnumInstsToMod.end(),
instToInsert) == pass.switchEnumInstsToMod.end()) {
@@ -1098,10 +1127,10 @@ void LoadableStorageAllocation::replaceLoadWithCopyAddr(
optimizableLoad->getParent()->erase(optimizableLoad);
}
static bool usesContainApplies(LoadInst *unoptimizableLoad,
irgen::IRGenModule &Mod,
LargeSILTypeMapper &Mapper) {
for (auto *user : unoptimizableLoad->getUses()) {
/// Does the value's uses contain instructions that *must* be rewrites?
static bool hasMandatoryRewriteUse(StructLoweringState &pass,
SILValue value) {
for (auto *user : value->getUses()) {
SILInstruction *userIns = user->getUser();
switch (userIns->getKind()) {
case SILInstructionKind::ApplyInst:
@@ -1110,13 +1139,11 @@ static bool usesContainApplies(LoadInst *unoptimizableLoad,
case SILInstructionKind::PartialApplyInst: {
ApplySite site(userIns);
SILValue callee = site.getCallee();
if (callee == unoptimizableLoad) {
if (callee == value) {
break;
}
SILType currType = unoptimizableLoad->getType().getObjectType();
GenericEnvironment *genEnv =
unoptimizableLoad->getFunction()->getGenericEnvironment();
SILType newSILType = Mapper.getNewSILType(genEnv, currType, Mod);
SILType currType = value->getType().getObjectType();
SILType newSILType = pass.getNewSILType(currType);
if (currType == newSILType) {
break;
}
@@ -1131,27 +1158,20 @@ static bool usesContainApplies(LoadInst *unoptimizableLoad,
void LoadableStorageAllocation::replaceLoadWithCopyAddrForModifiable(
LoadInst *unoptimizableLoad) {
if (!usesContainApplies(unoptimizableLoad, pass.Mod, pass.Mapper)) {
if (!hasMandatoryRewriteUse(pass, unoptimizableLoad)) {
return;
}
SILValue value = unoptimizableLoad->getOperand();
SILBuilderWithScope allocBuilder(&*pass.F->begin());
AllocStackInst *allocInstr =
allocBuilder.createAllocStack(value.getLoc(), value->getType());
AllocStackInst *alloc = allocate(pass, value.getLoc(),
value->getType().getObjectType());
SILBuilderWithScope outlinedBuilder(unoptimizableLoad);
createOutlinedCopyCall(outlinedBuilder, value, allocInstr, pass);
createOutlinedCopyCall(outlinedBuilder, value, alloc, pass);
// Insert stack deallocations.
for (TermInst *termInst : pass.returnInsts) {
SILBuilderWithScope deallocBuilder(termInst);
deallocBuilder.createDeallocStack(allocInstr->getLoc(), allocInstr);
}
SmallVector<Operand *, 8> usersToMod;
for (auto *user : unoptimizableLoad->getUses()) {
SILInstruction *userIns = user->getUser();
SmallVector<Operand *, 8> usesToMod;
for (auto *use : unoptimizableLoad->getUses()) {
SILInstruction *userIns = use->getUser();
switch (userIns->getKind()) {
case SILInstructionKind::CopyAddrInst:
case SILInstructionKind::DeallocStackInst:
@@ -1169,10 +1189,7 @@ void LoadableStorageAllocation::replaceLoadWithCopyAddrForModifiable(
break;
}
SILType currType = unoptimizableLoad->getType().getObjectType();
GenericEnvironment *genEnv =
userIns->getFunction()->getGenericEnvironment();
SILType newSILType =
pass.Mapper.getNewSILType(genEnv, currType, pass.Mod);
SILType newSILType = pass.getNewSILType(currType);
if (currType == newSILType) {
break;
}
@@ -1180,63 +1197,58 @@ void LoadableStorageAllocation::replaceLoadWithCopyAddrForModifiable(
pass.applies.end()) {
pass.applies.push_back(userIns);
}
usersToMod.push_back(user);
usesToMod.push_back(use);
break;
}
case SILInstructionKind::RetainValueInst: {
auto *insToInsert = dyn_cast<RetainValueInst>(userIns);
assert(insToInsert && "Unexpected cast failure");
auto *insToInsert = cast<RetainValueInst>(userIns);
pass.retainInstsToMod.push_back(insToInsert);
usersToMod.push_back(user);
usesToMod.push_back(use);
break;
}
case SILInstructionKind::ReleaseValueInst: {
auto *insToInsert = dyn_cast<ReleaseValueInst>(userIns);
assert(insToInsert && "Unexpected cast failure");
auto *insToInsert = cast<ReleaseValueInst>(userIns);
pass.releaseInstsToMod.push_back(insToInsert);
usersToMod.push_back(user);
usesToMod.push_back(use);
break;
}
case SILInstructionKind::StoreInst: {
auto *insToInsert = dyn_cast<StoreInst>(userIns);
assert(insToInsert && "Unexpected cast failure");
auto *insToInsert = cast<StoreInst>(userIns);
pass.storeInstsToMod.push_back(insToInsert);
usersToMod.push_back(user);
usesToMod.push_back(use);
break;
}
case SILInstructionKind::DebugValueInst: {
auto *insToInsert = dyn_cast<DebugValueInst>(userIns);
assert(insToInsert && "Unexpected cast failure");
auto *insToInsert = cast<DebugValueInst>(userIns);
pass.debugInstsToMod.push_back(insToInsert);
usersToMod.push_back(user);
usesToMod.push_back(use);
break;
}
case SILInstructionKind::DestroyValueInst: {
auto *insToInsert = dyn_cast<DestroyValueInst>(userIns);
assert(insToInsert && "Unexpected cast failure");
auto *insToInsert = cast<DestroyValueInst>(userIns);
pass.destroyValueInstsToMod.push_back(insToInsert);
usersToMod.push_back(user);
usesToMod.push_back(use);
break;
}
case SILInstructionKind::StructExtractInst: {
auto *instToInsert = dyn_cast<StructExtractInst>(userIns);
auto *instToInsert = cast<StructExtractInst>(userIns);
pass.structExtractInstsToMod.push_back(instToInsert);
usersToMod.push_back(user);
usesToMod.push_back(use);
break;
}
case SILInstructionKind::SwitchEnumInst: {
auto *instToInsert = dyn_cast<SwitchEnumInst>(userIns);
auto *instToInsert = cast<SwitchEnumInst>(userIns);
pass.switchEnumInstsToMod.push_back(instToInsert);
usersToMod.push_back(user);
usesToMod.push_back(use);
break;
}
default:
break;
}
}
while (!usersToMod.empty()) {
auto *currUser = usersToMod.pop_back_val();
currUser->set(allocInstr);
while (!usesToMod.empty()) {
auto *use = usesToMod.pop_back_val();
use->set(alloc);
}
}
@@ -1313,16 +1325,13 @@ void LoadableStorageAllocation::convertIndirectFunctionArgs() {
SILBasicBlock *entry = pass.F->getEntryBlock();
SILBuilderWithScope argBuilder(entry->begin());
GenericEnvironment *genEnv = pass.F->getGenericEnvironment();
for (SILArgument *arg : entry->getArguments()) {
SILType storageType = arg->getType();
SILType newSILType =
pass.Mapper.getNewSILType(genEnv, storageType, pass.Mod);
SILType newSILType = pass.getNewSILType(storageType);
if (newSILType != storageType) {
ValueOwnershipKind ownership = arg->getOwnershipKind();
arg = replaceArgType(argBuilder, arg, newSILType);
if (isLargeLoadableType(genEnv, storageType, pass.Mod)) {
if (pass.isLargeLoadableType(storageType)) {
// Add to largeLoadableArgs if and only if it wasn't a modified function
// signature arg
pass.largeLoadableArgs.push_back(arg);
@@ -1372,7 +1381,7 @@ void LoadableStorageAllocation::convertApplyResults() {
continue;
}
auto resultStorageType = origSILFunctionType->getAllResultsType();
if (!isLargeLoadableType(genEnv, resultStorageType, pass.Mod)) {
if (!pass.isLargeLoadableType(resultStorageType)) {
// Make sure it contains a function type
auto numFuncTy = llvm::count_if(origSILFunctionType->getResults(),
[](const SILResultInfo &origResult) {
@@ -1393,8 +1402,7 @@ void LoadableStorageAllocation::convertApplyResults() {
(void)numFuncTy;
continue;
}
auto newSILType =
pass.Mapper.getNewSILType(genEnv, resultStorageType, pass.Mod);
auto newSILType = pass.getNewSILType(resultStorageType);
auto *newVal = allocateForApply(currIns, newSILType.getObjectType());
if (auto apply = dyn_cast<ApplyInst>(currIns)) {
apply->replaceAllUsesWith(newVal);
@@ -1422,8 +1430,7 @@ void LoadableStorageAllocation::
for (SILArgument *arg : entry->getArguments()) {
SILType storageType = arg->getType();
GenericEnvironment *genEnv = pass.F->getGenericEnvironment();
SILType newSILType =
pass.Mapper.getNewSILType(genEnv, storageType, pass.Mod);
SILType newSILType = pass.getNewSILType(storageType);
if (containsFunctionSignature(genEnv, pass.Mod, storageType, newSILType)) {
auto *castInstr = argBuilder.createUncheckedBitCast(
RegularLocation(const_cast<ValueDecl *>(arg->getDecl())), arg,
@@ -1436,7 +1443,6 @@ void LoadableStorageAllocation::
void LoadableStorageAllocation::convertIndirectBasicBlockArgs() {
SILBasicBlock *entry = pass.F->getEntryBlock();
GenericEnvironment *genEnv = pass.F->getGenericEnvironment();
for (SILBasicBlock &BB : *pass.F) {
if (&BB == entry) {
// Already took care of function args
@@ -1448,8 +1454,7 @@ void LoadableStorageAllocation::convertIndirectBasicBlockArgs() {
continue;
}
SILType storageType = arg->getType();
SILType newSILType =
pass.Mapper.getNewSILType(genEnv, storageType, pass.Mod);
SILType newSILType = pass.getNewSILType(storageType);
convertBBArgType(argBuilder, newSILType, arg);
}
}
@@ -1584,111 +1589,86 @@ private:
};
} // end anonymous namespace
static void setInstrUsers(StructLoweringState &pass, AllocStackInst *allocInstr,
SILValue instrOperand, StoreInst *store) {
SmallVector<Operand *, 8> uses(instrOperand->getUses());
for (Operand *userOp : uses) {
SILInstruction *user = userOp->getUser();
if (user == store) {
continue;
}
/// Given that we've allocated space to hold a scalar value, try to rewrite
/// the uses of the scalar to be uses of the address.
static void rewriteUsesOfSscalar(StructLoweringState &pass,
SILValue address, SILValue scalar,
StoreInst *storeToAddress) {
// Copy the uses, since we're going to edit them.
SmallVector<Operand *, 8> uses(scalar->getUses());
for (Operand *scalarUse : uses) {
SILInstruction *user = scalarUse->getUser();
if (ApplySite::isa(user)) {
ApplySite site(user);
if (modifiableApply(site, pass.Mod)) {
userOp->set(allocInstr);
// Just rewrite the operand in-place. This will produce a temporary
// type error, but we should fix that up when we rewrite the apply's
// function type.
scalarUse->set(address);
}
} else if (isa<YieldInst>(user)) {
// The rules for the yield are changing anyway, so we can just rewrite
// it in-place.
scalarUse->set(address);
} else if (auto *storeUser = dyn_cast<StoreInst>(user)) {
// Don't rewrite the store to the allocation.
if (storeUser == storeToAddress)
continue;
// Optimization: replace with copy_addr to reduce code size
assert(std::find(pass.storeInstsToMod.begin(), pass.storeInstsToMod.end(),
storeUser) == pass.storeInstsToMod.end() &&
"Did not expect this instr in storeInstsToMod");
SILBuilderWithScope copyBuilder(storeUser);
SILValue tgt = storeUser->getDest();
createOutlinedCopyCall(copyBuilder, allocInstr, tgt, pass);
SILValue dest = storeUser->getDest();
createOutlinedCopyCall(copyBuilder, address, dest, pass);
storeUser->eraseFromParent();
} else if (auto *dbgInst = dyn_cast<DebugValueInst>(user)) {
SILBuilderWithScope dbgBuilder(dbgInst);
// Rewrite the debug_value to point to the variable in the alloca.
dbgBuilder.createDebugValueAddr(dbgInst->getLoc(), allocInstr,
dbgBuilder.createDebugValueAddr(dbgInst->getLoc(), address,
*dbgInst->getVarInfo());
dbgInst->eraseFromParent();
}
}
}
static void allocateAndSetForInstrOperand(StructLoweringState &pass,
SingleValueInstruction *instrOperand){
assert(instrOperand->getType().isObject());
SILBuilderWithScope allocBuilder(&*pass.F->begin());
AllocStackInst *allocInstr = allocBuilder.createAllocStack(
instrOperand->getLoc(), instrOperand->getType());
static void allocateAndSetForInstResult(StructLoweringState &pass,
SILValue instResult,
SILInstruction *inst) {
auto alloc = allocate(pass, inst->getLoc(), instResult->getType());
auto II = instrOperand->getIterator();
auto II = inst->getIterator();
++II;
SILBuilderWithScope storeBuilder(II);
StoreInst *store = nullptr;
if (pass.F->hasQualifiedOwnership()) {
store = storeBuilder.createStore(instrOperand->getLoc(), instrOperand,
allocInstr, StoreOwnershipQualifier::Init);
} else {
store = storeBuilder.createStore(instrOperand->getLoc(), instrOperand,
allocInstr,
StoreOwnershipQualifier::Unqualified);
}
auto store = createStoreInit(pass, II, inst->getLoc(), instResult, alloc);
// Insert stack deallocations.
for (TermInst *termInst : pass.returnInsts) {
SILBuilderWithScope deallocBuilder(termInst);
deallocBuilder.createDeallocStack(allocInstr->getLoc(), allocInstr);
}
// Traverse all the uses of instrOperand - see if we can replace
setInstrUsers(pass, allocInstr, instrOperand, store);
// Traverse all the uses of instResult - see if we can replace
rewriteUsesOfSscalar(pass, alloc, instResult, store);
}
static void allocateAndSetForArgumentOperand(StructLoweringState &pass,
SILValue value,
SILInstruction *applyInst) {
assert(value->getType().isObject());
auto *arg = dyn_cast<SILArgument>(value);
assert(arg && "non-instr operand must be an argument");
static void allocateAndSetForArgument(StructLoweringState &pass,
SILArgument *value,
SILInstruction *user) {
AllocStackInst *alloc = allocate(pass, user->getLoc(), value->getType());
SILBuilderWithScope allocBuilder(&*pass.F->begin());
AllocStackInst *allocInstr =
allocBuilder.createAllocStack(applyInst->getLoc(), value->getType());
SILLocation loc = user->getLoc();
loc.markAutoGenerated();
auto storeIt = arg->getParent()->begin();
if (storeIt == pass.F->begin()->begin()) {
// Store should happen *after* allocInstr
++storeIt;
// Store the value into the allocation.
auto II = value->getParent()->begin();
if (II == alloc->getParent()->begin()) {
// Store should happen *after* the allocation.
++II;
}
SILBuilderWithScope storeBuilder(storeIt);
SILLocation Loc = applyInst->getLoc();
Loc.markAutoGenerated();
auto store = createStoreInit(pass, II, loc, value, alloc);
StoreInst *store = nullptr;
if (pass.F->hasQualifiedOwnership()) {
store = storeBuilder.createStore(Loc, value, allocInstr,
StoreOwnershipQualifier::Init);
} else {
store = storeBuilder.createStore(Loc, value, allocInstr,
StoreOwnershipQualifier::Unqualified);
}
// Insert stack deallocations.
for (TermInst *termInst : pass.returnInsts) {
SILBuilderWithScope deallocBuilder(termInst);
deallocBuilder.createDeallocStack(allocInstr->getLoc(), allocInstr);
}
// Traverse all the uses of instrOperand - see if we can replace
setInstrUsers(pass, allocInstr, value, store);
// Traverse all the uses of value - see if we can replace
rewriteUsesOfSscalar(pass, alloc, value, store);
}
static bool allUsesAreReplaceable(SingleValueInstruction *instr,
irgen::IRGenModule &Mod,
LargeSILTypeMapper &Mapper) {
bool allUsesAreReplaceable = true;
static bool allUsesAreReplaceable(StructLoweringState &pass,
SingleValueInstruction *instr) {
for (auto *user : instr->getUses()) {
SILInstruction *userIns = user->getUser();
switch (userIns->getKind()) {
@@ -1704,32 +1684,68 @@ static bool allUsesAreReplaceable(SingleValueInstruction *instr,
case SILInstructionKind::PartialApplyInst: {
// Replaceable only if it is not the function pointer
ApplySite site(userIns);
if (!modifiableApply(site, Mod)) {
allUsesAreReplaceable = false;
break;
if (!modifiableApply(site, pass.Mod)) {
return false;
}
SILValue callee = site.getCallee();
if (callee == instr) {
allUsesAreReplaceable = false;
return false;
}
SILType currType = instr->getType().getObjectType();
GenericEnvironment *genEnv =
instr->getFunction()->getGenericEnvironment();
SILType newSILType = Mapper.getNewSILType(genEnv, currType, Mod);
SILType newSILType = pass.getNewSILType(currType);
if (currType == newSILType) {
allUsesAreReplaceable = false;
return false;
}
break;
}
case SILInstructionKind::StructExtractInst:
case SILInstructionKind::SwitchEnumInst: {
case SILInstructionKind::SwitchEnumInst:
break;
}
default:
allUsesAreReplaceable = false;
return false;
}
}
return true;
}
void LoadableStorageAllocation::replaceLoad(LoadInst *load) {
if (allUsesAreReplaceable(pass, load)) {
replaceLoadWithCopyAddr(load);
} else {
replaceLoadWithCopyAddrForModifiable(load);
}
}
static void allocateAndSet(StructLoweringState &pass,
LoadableStorageAllocation &allocator,
SILValue operand, SILInstruction *user) {
auto inst = operand->getDefiningInstruction();
if (!inst) {
allocateAndSetForArgument(pass, cast<SILArgument>(operand), user);
return;
}
if (auto *load = dyn_cast<LoadInst>(operand)) {
allocator.replaceLoad(load);
} else {
// TODO: peephole: special handling of known cases:
// ApplyInst, TupleExtractInst
allocateAndSetForInstResult(pass, operand, inst);
}
}
/// Rewrite all of the large-loadable operands in the given list.
static void allocateAndSetAll(StructLoweringState &pass,
LoadableStorageAllocation &allocator,
SILInstruction *user,
MutableArrayRef<Operand> operands) {
for (Operand &operand : operands) {
SILValue value = operand.get();
SILType silType = value->getType();
if (pass.isLargeLoadableType(silType)) {
allocateAndSet(pass, allocator, value, user);
}
}
return allUsesAreReplaceable;
}
static void castTupleInstr(SingleValueInstruction *instr, IRGenModule &Mod,
@@ -1774,38 +1790,18 @@ static SILValue createCopyOfEnum(StructLoweringState &pass,
auto value = orig->getOperand();
auto type = value->getType();
if (type.isObject()) {
SILBuilderWithScope allocBuilder(&*pass.F->begin());
// support for non-address operands / enums
auto *allocInstr = allocBuilder.createAllocStack(orig->getLoc(), type);
SILBuilderWithScope storeBuilder(orig);
StoreInst *store = nullptr;
if (pass.F->hasQualifiedOwnership()) {
store = storeBuilder.createStore(orig->getLoc(), value, allocInstr,
StoreOwnershipQualifier::Init);
} else {
store = storeBuilder.createStore(orig->getLoc(), value, allocInstr,
StoreOwnershipQualifier::Unqualified);
}
// Insert stack deallocations.
for (TermInst *termInst : pass.returnInsts) {
SILBuilderWithScope deallocBuilder(termInst);
deallocBuilder.createDeallocStack(allocInstr->getLoc(), allocInstr);
}
value = allocInstr;
auto *alloc = allocate(pass, orig->getLoc(), type);
createStoreInit(pass, orig->getIterator(), orig->getLoc(), value, alloc);
return alloc;
}
SILBuilderWithScope allocBuilder(&*pass.F->begin());
auto *allocInstr = allocBuilder.createAllocStack(value.getLoc(), type);
auto alloc = allocate(pass, value.getLoc(), type.getObjectType());
SILBuilderWithScope copyBuilder(orig);
createOutlinedCopyCall(copyBuilder, value, allocInstr, pass);
createOutlinedCopyCall(copyBuilder, value, alloc, pass);
for (TermInst *termInst : pass.returnInsts) {
SILBuilderWithScope deallocBuilder(termInst);
deallocBuilder.createDeallocStack(allocInstr->getLoc(), allocInstr);
}
return allocInstr;
return alloc;
}
static void createResultTyInstrAndLoad(LoadableStorageAllocation &allocator,
@@ -1839,11 +1835,8 @@ static void createResultTyInstrAndLoad(LoadableStorageAllocation &allocator,
return;
}
if (allUsesAreReplaceable(loadArg, pass.Mod, pass.Mapper)) {
allocator.replaceLoadWithCopyAddr(loadArg);
} else {
allocator.replaceLoadWithCopyAddrForModifiable(loadArg);
}
allocator.replaceLoad(loadArg);
if (updateResultTy) {
pass.resultTyInstsToMod.insert(newInstr);
}
@@ -1852,8 +1845,6 @@ static void createResultTyInstrAndLoad(LoadableStorageAllocation &allocator,
static void rewriteFunction(StructLoweringState &pass,
LoadableStorageAllocation &allocator) {
GenericEnvironment *genEnv = pass.F->getGenericEnvironment();
bool repeat = false;
llvm::SetVector<SILInstruction *> currentModApplies;
do {
@@ -1873,8 +1864,7 @@ static void rewriteFunction(StructLoweringState &pass,
EnumElementDecl *decl = currCase.first;
for (SILArgument *arg : currBB->getArguments()) {
SILType storageType = arg->getType();
SILType newSILType =
pass.Mapper.getNewSILType(genEnv, storageType, pass.Mod);
SILType newSILType = pass.getNewSILType(storageType);
if (storageType == newSILType) {
newSILType = newSILType.getAddressType();
}
@@ -1901,11 +1891,7 @@ static void rewriteFunction(StructLoweringState &pass,
continue;
}
if (allUsesAreReplaceable(loadArg, pass.Mod, pass.Mapper)) {
allocator.replaceLoadWithCopyAddr(loadArg);
} else {
allocator.replaceLoadWithCopyAddrForModifiable(loadArg);
}
allocator.replaceLoad(loadArg);
}
caseBBs.push_back(std::make_pair(decl, currBB));
}
@@ -1927,33 +1913,10 @@ static void rewriteFunction(StructLoweringState &pass,
currentModApplies.insert(applyInst);
}
ApplySite applySite = ApplySite(applyInst);
for (Operand &operand : applySite.getArgumentOperands()) {
SILValue currOperand = operand.get();
SILType silType = currOperand->getType();
if (isLargeLoadableType(genEnv, silType, pass.Mod)) {
auto currOperandInstr = dyn_cast<SingleValueInstruction>(currOperand);
// Get its storage location as a new operand
if (!currOperandInstr) {
allocateAndSetForArgumentOperand(pass, currOperand, applyInst);
} else if (auto *load = dyn_cast<LoadInst>(currOperandInstr)) {
// If the load is of a function type - do not replace it.
if (isFuncOrOptionalFuncType(load->getType())) {
continue;
}
if (allUsesAreReplaceable(load, pass.Mod, pass.Mapper)) {
allocator.replaceLoadWithCopyAddr(load);
} else {
allocator.replaceLoadWithCopyAddrForModifiable(load);
}
} else {
// TODO: peephole: special handling of known cases:
// ApplyInst, TupleExtractInst
allocateAndSetForInstrOperand(pass, currOperandInstr);
}
}
}
allocateAndSetAll(pass, allocator, applyInst,
applySite.getArgumentOperands());
}
repeat = !pass.switchEnumInstsToMod.empty() ||
!pass.structExtractInstsToMod.empty();
assert(pass.applies.empty());
@@ -1982,8 +1945,7 @@ static void rewriteFunction(StructLoweringState &pass,
auto *instr = pass.allocStackInstsToMod.pop_back_val();
SILBuilderWithScope allocBuilder(instr);
SILType currSILType = instr->getType();
SILType newSILType =
pass.Mapper.getNewSILType(genEnv, currSILType, pass.Mod);
SILType newSILType = pass.getNewSILType(currSILType);
auto *newInstr = allocBuilder.createAllocStack(instr->getLoc(), newSILType,
instr->getVarInfo());
instr->replaceAllUsesWith(newInstr);
@@ -1994,8 +1956,7 @@ static void rewriteFunction(StructLoweringState &pass,
auto *instr = pass.pointerToAddrkInstsToMod.pop_back_val();
SILBuilderWithScope pointerBuilder(instr);
SILType currSILType = instr->getType();
SILType newSILType =
pass.Mapper.getNewSILType(genEnv, currSILType, pass.Mod);
SILType newSILType = pass.getNewSILType(currSILType);
auto *newInstr = pointerBuilder.createPointerToAddress(
instr->getLoc(), instr->getOperand(), newSILType.getAddressType(),
instr->isStrict());
@@ -2071,8 +2032,7 @@ static void rewriteFunction(StructLoweringState &pass,
// Update the return type of these instrs
// Note: The operand was already updated!
SILType currSILType = instr->getType().getObjectType();
SILType newSILType =
pass.Mapper.getNewSILType(genEnv, currSILType, pass.Mod);
SILType newSILType = pass.getNewSILType(currSILType);
SILBuilderWithScope resultTyBuilder(instr);
SILLocation Loc = instr->getLoc();
SingleValueInstruction *newInstr = nullptr;
@@ -2206,13 +2166,8 @@ static void rewriteFunction(StructLoweringState &pass,
SILBuilderWithScope retCopyBuilder(II);
createOutlinedCopyCall(retCopyBuilder, retOp, retArg, pass, &regLoc);
} else {
if (pass.F->hasQualifiedOwnership()) {
retBuilder.createStore(regLoc, retOp, retArg,
StoreOwnershipQualifier::Init);
} else {
retBuilder.createStore(regLoc, retOp, retArg,
StoreOwnershipQualifier::Unqualified);
}
retBuilder.createStore(regLoc, retOp, retArg,
getStoreInitOwnership(pass, retOp->getType()));
}
auto emptyTy = retBuilder.getModule().Types.getLoweredType(
TupleType::getEmpty(retBuilder.getModule().getASTContext()));
@@ -2230,9 +2185,9 @@ static bool rewriteFunctionReturn(StructLoweringState &pass) {
auto loweredTy = pass.F->getLoweredFunctionType();
SILFunction *F = pass.F;
SILType resultTy = loweredTy->getAllResultsType();
SILType newSILType = pass.Mapper.getNewSILType(genEnv, resultTy, pass.Mod);
SILType newSILType = pass.getNewSILType(resultTy);
// We (currently) only care about function signatures
if (isLargeLoadableType(genEnv, resultTy, pass.Mod)) {
if (pass.isLargeLoadableType(resultTy)) {
return true;
} else if (containsFunctionSignature(genEnv, pass.Mod, resultTy,
newSILType) &&