averello/swift-mirror
1
0
Fork 0
mirror of https://github.com/apple/swift.git synced 2025-12-14 20:36:38 +01:00
Code Issues Packages Projects Releases Wiki Activity
Files
2deb2f9a87352dd6a3d63f90d0f292289cb15ee4
swift-mirror/lib/SILOptimizer/Transforms/SILCodeMotion.cpp
Erik Eckstein b80a08ee6c CodeMotion: check the ownership of arguments when sinking arguments 
So far we only checked the ownership of incoming values.
But even if the incoming instruction has no ownership, the argument may have.
This can happen with enums which are constructed with a non-payload case:

   %1 = enum $Optional<C>, #Optional.none!enumelt
   br bb3(%1)
 bb1(%3 : @owned $Optional<C>):

Fixes an ownership verification error:
rdar://142506300
2025-01-09 19:56:25 +01:00

1840 lines
65 KiB
C++
Raw Blame History

//===--- SILCodeMotion.cpp - Code Motion Optimizations --------------------===//
//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See https://swift.org/LICENSE.txt for license information
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
//
//===----------------------------------------------------------------------===//
#define DEBUG_TYPE "sil-codemotion"
#include "swift/AST/Module.h"
#include "swift/Basic/Assertions.h"
#include "swift/Basic/BlotMapVector.h"
#include "swift/SIL/DebugUtils.h"
#include "swift/SIL/SILBuilder.h"
#include "swift/SIL/SILModule.h"
#include "swift/SIL/SILType.h"
#include "swift/SIL/SILValue.h"
#include "swift/SIL/SILVisitor.h"
#include "swift/SILOptimizer/Analysis/ARCAnalysis.h"
#include "swift/SILOptimizer/Analysis/AliasAnalysis.h"
#include "swift/SILOptimizer/Analysis/PostOrderAnalysis.h"
#include "swift/SILOptimizer/Analysis/RCIdentityAnalysis.h"
#include "swift/SILOptimizer/PassManager/Passes.h"
#include "swift/SILOptimizer/PassManager/Transforms.h"
#include "swift/SILOptimizer/Utils/InstOptUtils.h"
#include "swift/SILOptimizer/Utils/OwnershipOptUtils.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include <optional>
STATISTIC(NumSunk, "Number of instructions sunk");
STATISTIC(NumRefCountOpsSimplified, "Number of enum ref count ops simplified");
STATISTIC(NumHoisted, "Number of instructions hoisted");
STATISTIC(NumSILArgumentReleaseHoisted, "Number of silargument release instructions hoisted");
llvm::cl::opt<bool> DisableSILRRCodeMotion("disable-sil-cm-rr-cm", llvm::cl::init(true));
using namespace swift;
namespace {
//===----------------------------------------------------------------------===//
// Utility
//===----------------------------------------------------------------------===//
static void createRefCountOpForPayload(SILBuilder &Builder, SILInstruction *I,
EnumElementDecl *EnumDecl,
SILValue DefOfEnum = SILValue()) {
assert(EnumDecl->hasAssociatedValues() &&
"We assume enumdecl has an argument type");
SILModule &Mod = I->getModule();
// The enum value is either passed as an extra argument if we are moving an
// retain that does not refer to the enum typed value - otherwise it is the
// argument to the refcount instruction.
SILValue EnumVal = DefOfEnum ? DefOfEnum : I->getOperand(0);
SILType ArgType = EnumVal->getType().getEnumElementType(
EnumDecl, Mod, TypeExpansionContext(Builder.getFunction()));
auto *UEDI =
Builder.createUncheckedEnumData(I->getLoc(), EnumVal, EnumDecl, ArgType);
SILType UEDITy = UEDI->getType();
// If our payload is trivial, we do not need to insert any retain or release
// operations.
if (UEDITy.isTrivial(*I->getFunction()))
return;
++NumRefCountOpsSimplified;
// If we have a retain value...
if (auto RCI = dyn_cast<RetainValueInst>(I)) {
// And our payload is refcounted, insert a strong_retain onto the
// payload.
if (UEDITy.isReferenceCounted(Mod)) {
Builder.createStrongRetain(I->getLoc(), UEDI, RCI->getAtomicity());
return;
}
// Otherwise, insert a retain_value on the payload.
Builder.createRetainValue(I->getLoc(), UEDI, RCI->getAtomicity());
return;
}
// At this point we know that we must have a release_value and a non-trivial
// payload.
assert(isa<ReleaseValueInst>(I) && "If I is not a retain value here, it must "
"be a release value since enums do not have reference semantics.");
auto *RCI = cast<ReleaseValueInst>(I);
// If our payload has reference semantics, insert the strong release.
if (UEDITy.isReferenceCounted(Mod)) {
Builder.createStrongRelease(I->getLoc(), UEDI, RCI->getAtomicity());
return;
}
// Otherwise if our payload is non-trivial but lacking reference semantics,
// insert the release_value.
Builder.createReleaseValue(I->getLoc(), UEDI, RCI->getAtomicity());
}
//===----------------------------------------------------------------------===//
// Enum Tag Dataflow
//===----------------------------------------------------------------------===//
namespace {
class EnumCaseDataflowContext;
using EnumBBCaseList =
llvm::SmallVector<std::pair<SILBasicBlock *, EnumElementDecl *>, 2>;
/// Class that performs enum tag state dataflow on the given BB.
class BBEnumTagDataflowState
: public SILInstructionVisitor<BBEnumTagDataflowState, bool> {
EnumCaseDataflowContext *Context;
NullablePtr<SILBasicBlock> BB;
SmallBlotMapVector<unsigned, EnumElementDecl *, 4> ValueToCaseMap;
SmallBlotMapVector<unsigned, EnumBBCaseList, 4> EnumToEnumBBCaseListMap;
public:
BBEnumTagDataflowState() = default;
BBEnumTagDataflowState(const BBEnumTagDataflowState &Other) = default;
~BBEnumTagDataflowState() = default;
SWIFT_DEBUG_DUMP;
bool init(EnumCaseDataflowContext &Context, SILBasicBlock *NewBB);
SILBasicBlock *getBB() { return BB.get(); }
using iterator = decltype(ValueToCaseMap)::iterator;
iterator begin() { return ValueToCaseMap.getItems().begin(); }
iterator end() { return ValueToCaseMap.getItems().end(); }
void clear() { ValueToCaseMap.clear(); }
bool visitSILInstruction(SILInstruction *I) { return false; }
bool visitEnumInst(EnumInst *EI);
bool visitUncheckedEnumDataInst(UncheckedEnumDataInst *UEDI);
bool visitRetainValueInst(RetainValueInst *RVI);
bool visitReleaseValueInst(ReleaseValueInst *RVI);
bool process();
bool hoistDecrementsIntoSwitchRegions(AliasAnalysis *AA);
bool sinkIncrementsOutOfSwitchRegions(AliasAnalysis *AA,
RCIdentityFunctionInfo *RCIA);
void handlePredSwitchEnum(SwitchEnumInst *S);
void handlePredCondSelectEnum(CondBranchInst *CondBr);
/// Helper method which initializes this state map with the data from the
/// first predecessor BB.
///
/// We will be performing an intersection in a later step of the merging.
bool initWithFirstPred(SILBasicBlock *FirstPredBB);
/// Top level merging function for predecessors.
void mergePredecessorStates();
/// If we have a single predecessor, see if the predecessor's terminator was a
/// switch_enum or a (cond_br + select_enum). If so, track in this block the
/// enum state of the operand.
void mergeSinglePredTermInfoIntoState(SILBasicBlock *Pred);
private:
EnumCaseDataflowContext &getContext() const;
unsigned getIDForValue(SILValue V) const;
SILValue getValueForID(unsigned ID) const;
};
/// Map all blocks to BBEnumTagDataflowState in RPO order.
class EnumCaseDataflowContext {
PostOrderFunctionInfo *PO;
std::vector<BBEnumTagDataflowState> BBToStateVec;
std::vector<SILValue> IDToEnumValueMap;
llvm::DenseMap<SILValue, unsigned> EnumValueToIDMap;
public:
EnumCaseDataflowContext(PostOrderFunctionInfo *PO) : PO(PO), BBToStateVec() {
BBToStateVec.resize(PO->size());
unsigned RPOIdx = 0;
for (SILBasicBlock *BB : PO->getReversePostOrder()) {
BBToStateVec[RPOIdx].init(*this, BB);
++RPOIdx;
}
}
/// Return true if we have a valid value for the given ID;
bool hasValueForID(unsigned ID) const {
return ID < IDToEnumValueMap.size() && IDToEnumValueMap[ID];
}
SILValue getValueForID(unsigned ID) const { return IDToEnumValueMap[ID]; }
unsigned getIDForValue(SILValue V) const {
// We are being a little tricky here. Here is what is happening:
//
// 1. When we start we do not know if we are already tracking V, so we
// prepare /as if/ V was new.
//
// 2. When we perform the insertion, if V already has a value associated
// with it, we return the iterator to that value. The iterator contains
// inside of it the actual index.
//
// 3. Otherwise, we initialize V in EnumValueToIDMap with NewID. Rather than
// re-accessing the iterator, we just return the value we have already
// computed.
unsigned NewID = IDToEnumValueMap.size();
auto &This = const_cast<EnumCaseDataflowContext &>(*this);
auto Iter = This.EnumValueToIDMap.insert({V, NewID});
if (!Iter.second)
return Iter.first->second;
This.IDToEnumValueMap.emplace_back(V);
return NewID;
}
void blotValue(SILValue V) {
unsigned ID = getIDForValue(V);
IDToEnumValueMap[ID] = SILValue();
}
unsigned size() const { return BBToStateVec.size(); }
BBEnumTagDataflowState &getRPOState(unsigned RPOIdx) {
return BBToStateVec[RPOIdx];
}
/// \return BBEnumTagDataflowState or NULL for unreachable blocks.
BBEnumTagDataflowState *getBBState(SILBasicBlock *BB) {
if (auto ID = PO->getRPONumber(BB)) {
return &getRPOState(*ID);
}
return nullptr;
}
};
} // end anonymous namespace
EnumCaseDataflowContext &BBEnumTagDataflowState::getContext() const {
// Context and BB are initialized together, so we only need to check one.
assert(BB.isNonNull() && "Uninitialized state?!");
return *Context;
}
unsigned BBEnumTagDataflowState::getIDForValue(SILValue V) const {
return getContext().getIDForValue(V);
}
SILValue BBEnumTagDataflowState::getValueForID(unsigned ID) const {
return getContext().getValueForID(ID);
}
void BBEnumTagDataflowState::handlePredSwitchEnum(SwitchEnumInst *S) {
// Find the tag associated with our BB and set the state of the
// enum we switch on to that value. This is important so we can determine
// covering switches for enums that have cases without payload.
// Next check if we are the target of a default switch_enum case. If we are,
// no interesting information can be extracted, so bail...
if (S->hasDefault() && S->getDefaultBB() == getBB())
return;
// Otherwise, attempt to find the tag associated with this BB in the switch
// enum...
for (unsigned i = 0, e = S->getNumCases(); i != e; ++i) {
auto P = S->getCase(i);
// If this case of the switch is not matched up with this BB, skip the
// case...
if (P.second != getBB())
continue;
// Ok, we found the case for our BB. If we don't have an enum tag (which can
// happen if we have a default statement), return. There is nothing more we
// can do.
if (!P.first)
return;
// Ok, we have a matching BB and a matching enum tag. Set the state and
// return.
ValueToCaseMap[getIDForValue(S->getOperand())] = P.first;
return;
}
llvm_unreachable("A successor of a switch_enum terminated BB should be in "
"the switch_enum.");
}
void BBEnumTagDataflowState::handlePredCondSelectEnum(CondBranchInst *CondBr) {
auto *EITI = dyn_cast<SelectEnumInst>(CondBr->getCondition());
if (!EITI)
return;
NullablePtr<EnumElementDecl> TrueElement = EITI->getSingleTrueElement();
if (TrueElement.isNull())
return;
// Find the tag associated with our BB and set the state of the
// enum we switch on to that value. This is important so we can determine
// covering switches for enums that have cases without payload.
// Check if we are the true case, ie, we know that we are the given tag.
const auto &Operand = EITI->getEnumOperand();
if (CondBr->getTrueBB() == getBB()) {
ValueToCaseMap[getIDForValue(Operand)] = TrueElement.get();
return;
}
// If the enum only has 2 values and its tag isn't the true branch, then we
// know the true branch must be the other tag.
if (EnumDecl *E = Operand->getType().getEnumOrBoundGenericEnum()) {
// We can't do this optimization on non-exhaustive enums.
const SILFunction *Fn = CondBr->getFunction();
bool IsExhaustive =
E->isEffectivelyExhaustive(Fn->getModule().getSwiftModule(),
Fn->getResilienceExpansion());
if (!IsExhaustive)
return;
// Look for a single other element on this enum.
EnumElementDecl *OtherElt = nullptr;
for (EnumElementDecl *Elt : E->getAllElements()) {
// Skip the case where we find the select_enum element
if (Elt == TrueElement.get())
continue;
// If we find another element, then we must have more than 2, so bail.
if (OtherElt)
return;
OtherElt = Elt;
}
// Only a single enum element? How would this even get here? We should
// handle it in SILCombine.
if (!OtherElt)
return;
// FIXME: Can we ever not be the false BB here?
if (CondBr->getTrueBB() != getBB()) {
ValueToCaseMap[getIDForValue(Operand)] = OtherElt;
return;
}
}
}
bool BBEnumTagDataflowState::initWithFirstPred(SILBasicBlock *FirstPredBB) {
// Try to look up the state for the first pred BB.
BBEnumTagDataflowState *FirstPredState = getContext().getBBState(FirstPredBB);
// If we fail, we found an unreachable block, bail.
if (FirstPredState == nullptr) {
LLVM_DEBUG(llvm::dbgs() << " Found an unreachable block!\n");
return false;
}
// Ok, our state is in the map, copy in the predecessors value to case map.
ValueToCaseMap = FirstPredState->ValueToCaseMap;
// If we are predecessors only successor, we can potentially hoist releases
// into it, so associate the first pred BB and the case for each value that we
// are tracking with it.
//
// TODO: I am writing this too fast. Clean this up later.
if (FirstPredBB->getSingleSuccessorBlock()) {
for (auto P : ValueToCaseMap.getItems()) {
if (!P.has_value())
continue;
EnumToEnumBBCaseListMap[P->first].push_back({FirstPredBB, P->second});
}
}
return true;
}
void BBEnumTagDataflowState::mergeSinglePredTermInfoIntoState(
SILBasicBlock *Pred) {
// Grab the terminator of our one predecessor and if it is a switch enum, mix
// it into this state.
TermInst *PredTerm = Pred->getTerminator();
if (auto *S = dyn_cast<SwitchEnumInst>(PredTerm)) {
handlePredSwitchEnum(S);
return;
}
auto *CondBr = dyn_cast<CondBranchInst>(PredTerm);
if (!CondBr)
return;
handlePredCondSelectEnum(CondBr);
}
void BBEnumTagDataflowState::mergePredecessorStates() {
// If we have no predecessors, there is nothing to do so return early...
if (getBB()->pred_empty()) {
LLVM_DEBUG(llvm::dbgs() << " No Preds.\n");
return;
}
auto PI = getBB()->pred_begin(), PE = getBB()->pred_end();
if (*PI == getBB()) {
LLVM_DEBUG(llvm::dbgs() << " Found a self loop. Bailing!\n");
return;
}
// Grab the first predecessor BB.
SILBasicBlock *FirstPred = *PI;
++PI;
// Attempt to initialize our state with our first predecessor's state by just
// copying. We will be doing an intersection with all of the other BB.
if (!initWithFirstPred(FirstPred))
return;
// If we only have one predecessor see if we can gain any information and or
// knowledge from the terminator of our one predecessor. There is nothing more
// that we can do, return.
//
// This enables us to get enum information from switch_enum and cond_br about
// the value that an enum can take in our block. This is a common case that
// comes up.
if (PI == PE) {
mergeSinglePredTermInfoIntoState(FirstPred);
return;
}
LLVM_DEBUG(llvm::dbgs() <<" Merging in rest of predecessors...\n");
// Enum values that while merging we found conflicting values for. We blot
// them after the loop in order to ensure that we can still find the ends of
// switch regions.
llvm::SmallVector<unsigned, 4> CurBBValuesToBlot;
// If we do not find state for a specific value in any of our predecessor BBs,
// we cannot be the end of a switch region since we cannot cover our
// predecessor BBs with enum decls. Blot after the loop.
llvm::SmallVector<unsigned, 4> PredBBValuesToBlot;
// And for each remaining predecessor...
do {
// If we loop on ourselves, bail...
if (*PI == getBB()) {
LLVM_DEBUG(llvm::dbgs() << " Found a self loop. Bailing!\n");
return;
}
// Grab the predecessors state...
SILBasicBlock *PredBB = *PI;
BBEnumTagDataflowState *PredBBState = getContext().getBBState(PredBB);
if (PredBBState == nullptr) {
LLVM_DEBUG(llvm::dbgs() << " Found an unreachable block!\n");
return;
}
++PI;
// Then for each (SILValue, Enum Tag) that we are tracking...
for (auto P : ValueToCaseMap.getItems()) {
// If this SILValue was blotted, there is nothing left to do, we found
// some sort of conflicting definition and are being conservative.
if (!P.has_value())
continue;
// Then attempt to look up the enum state associated in our SILValue in
// the predecessor we are processing.
auto PredIter = PredBBState->ValueToCaseMap.find(P->first);
// If we cannot find the state associated with this SILValue in this
// predecessor or the ID in the corresponding predecessor was blotted, we
// cannot find a covering switch for this BB or forward any enum tag
// information for this enum value.
if (PredIter == PredBBState->ValueToCaseMap.end() ||
!(*PredIter).has_value()) {
// Otherwise, we are conservative and do not forward the EnumTag that we
// are tracking. Blot it!
LLVM_DEBUG(llvm::dbgs() << " Blotting: " << P->first);
CurBBValuesToBlot.push_back(P->first);
PredBBValuesToBlot.push_back(P->first);
continue;
}
// Then try to lookup the actual value associated with the ID. If we do
// not find one, then the enum was destroyed by another part of the pass.
SILValue PredValue = getValueForID((*PredIter)->first);
if (!PredValue)
continue;
// Check if out predecessor has any other successors. If that is true we
// clear all the state since we cannot hoist safely.
if (!PredBB->getSingleSuccessorBlock()) {
EnumToEnumBBCaseListMap.clear();
LLVM_DEBUG(llvm::dbgs() << " Predecessor has other "
"successors. Clearing BB cast list map.\n");
} else {
// Otherwise, add this case to our predecessor case list. We will unique
// this after we have finished processing all predecessors.
auto Case = std::make_pair(PredBB, (*PredIter)->second);
EnumToEnumBBCaseListMap[(*PredIter)->first].push_back(Case);
}
// And the states match, the enum state propagates to this BB.
if ((*PredIter)->second == P->second)
continue;
// Otherwise, we are conservative and do not forward the EnumTag that we
// are tracking. Blot it!
LLVM_DEBUG(llvm::dbgs() << " Blotting: " << P->first);
CurBBValuesToBlot.push_back(P->first);
}
} while (PI != PE);
for (unsigned ID : CurBBValuesToBlot) {
ValueToCaseMap.erase(ID);
}
for (unsigned ID : PredBBValuesToBlot) {
EnumToEnumBBCaseListMap.erase(ID);
}
}
bool BBEnumTagDataflowState::visitEnumInst(EnumInst *EI) {
unsigned ID = getIDForValue(SILValue(EI));
LLVM_DEBUG(llvm::dbgs() << " Storing enum into map. ID: " << ID
<< ". Value: " << *EI);
ValueToCaseMap[ID] = EI->getElement();
return false;
}
bool BBEnumTagDataflowState::visitUncheckedEnumDataInst(
UncheckedEnumDataInst *UEDI) {
unsigned ID = getIDForValue(UEDI->getOperand());
LLVM_DEBUG(llvm::dbgs() << " Storing unchecked enum data into map. ID: "
<< ID << ". Value: " << *UEDI);
ValueToCaseMap[ID] = UEDI->getElement();
return false;
}
bool BBEnumTagDataflowState::visitRetainValueInst(RetainValueInst *RVI) {
auto FindResult = ValueToCaseMap.find(getIDForValue(RVI->getOperand()));
if (FindResult == ValueToCaseMap.end())
return false;
// If we do not have any argument, kill the retain_value.
if (!(*FindResult)->second->hasAssociatedValues()) {
RVI->eraseFromParent();
return true;
}
LLVM_DEBUG(llvm::dbgs() << " Found RetainValue: " << *RVI);
LLVM_DEBUG(llvm::dbgs() << " Paired to Enum Oracle: "
<< (*FindResult)->first);
SILBuilderWithScope Builder(RVI);
createRefCountOpForPayload(Builder, RVI, (*FindResult)->second);
RVI->eraseFromParent();
return true;
}
bool BBEnumTagDataflowState::visitReleaseValueInst(ReleaseValueInst *RVI) {
auto FindResult = ValueToCaseMap.find(getIDForValue(RVI->getOperand()));
if (FindResult == ValueToCaseMap.end())
return false;
// If the enum has a deinit, preserve the original release.
if (hasValueDeinit(RVI->getOperand()))
return false;
// If we do not have any argument, just delete the release value.
if (!(*FindResult)->second->hasAssociatedValues()) {
RVI->eraseFromParent();
return true;
}
LLVM_DEBUG(llvm::dbgs() << " Found ReleaseValue: " << *RVI);
LLVM_DEBUG(llvm::dbgs() << " Paired to Enum Oracle: "
<< (*FindResult)->first);
SILBuilderWithScope Builder(RVI);
createRefCountOpForPayload(Builder, RVI, (*FindResult)->second);
RVI->eraseFromParent();
return true;
}
bool BBEnumTagDataflowState::process() {
bool Changed = false;
auto SI = getBB()->begin();
while (SI != getBB()->end()) {
SILInstruction *I = &*SI;
++SI;
Changed |= visit(I);
}
return Changed;
}
bool BBEnumTagDataflowState::hoistDecrementsIntoSwitchRegions(
AliasAnalysis *AA) {
bool Changed = false;
unsigned NumPreds = std::distance(getBB()->pred_begin(), getBB()->pred_end());
for (auto II = getBB()->begin(), IE = getBB()->end(); II != IE;) {
auto *RVI = dyn_cast<ReleaseValueInst>(&*II);
++II;
// If this instruction is not a release, skip it...
if (!RVI)
continue;
LLVM_DEBUG(llvm::dbgs() << " Visiting release: " << *RVI);
// Grab the operand of the release value inst.
SILValue Op = RVI->getOperand();
// Lookup the [(BB, EnumTag)] list for this operand.
unsigned ID = getIDForValue(Op);
auto R = EnumToEnumBBCaseListMap.find(ID);
// If we don't have one, skip this release value inst.
if (R == EnumToEnumBBCaseListMap.end()) {
LLVM_DEBUG(llvm::dbgs() << " Could not find [(BB, EnumTag)] "
"list for release_value's operand. Bailing!\n");
continue;
}
// If the enum has a deinit, preserve the original release.
if (hasValueDeinit(Op))
return false;
auto &EnumBBCaseList = (*R)->second;
// If we don't have an enum tag for each predecessor of this BB, bail since
// we do not know how to handle that BB.
if (EnumBBCaseList.size() != NumPreds) {
LLVM_DEBUG(llvm::dbgs() << " Found [(BB, EnumTag)] list for "
"release_value's operand, but we do not have "
"an enum tag for each predecessor. Bailing!\n");
LLVM_DEBUG(llvm::dbgs() << " List:\n");
LLVM_DEBUG(for (auto P : EnumBBCaseList) {
llvm::dbgs() << " ";
P.second->dump(llvm::dbgs());
});
continue;
}
// Finally ensure that we have no users of this operand preceding the
// release_value in this BB. If we have users like that we cannot hoist the
// release past them unless we know that there is an additional set of
// releases that together post-dominate this release. If we cannot do this,
// skip this release.
//
// TODO: We need information from the ARC optimizer to prove that property
// if we are going to use it.
if (valueHasARCUsesInInstructionRange(Op, getBB()->begin(),
SILBasicBlock::iterator(RVI), AA)) {
LLVM_DEBUG(llvm::dbgs() << " Release value has use that stops "
"hoisting! Bailing!\n");
continue;
}
LLVM_DEBUG(llvm::dbgs() << " Its safe to perform the "
"transformation!\n");
// Otherwise perform the transformation.
for (auto P : EnumBBCaseList) {
// If we don't have an argument for this case, there is nothing to
// do... continue...
if (!P.second->hasAssociatedValues())
continue;
// Otherwise create the release_value before the terminator of the
// predecessor.
assert(P.first->getSingleSuccessorBlock() &&
"Cannot hoist release into BB that has multiple successors");
SILBuilderWithScope Builder(P.first->getTerminator(), RVI);
createRefCountOpForPayload(Builder, RVI, P.second);
}
RVI->eraseFromParent();
++NumHoisted;
Changed = true;
}
return Changed;
}
static SILInstruction *findLastSinkableMatchingEnumValueRCIncrementInPred(
AliasAnalysis *AA, RCIdentityFunctionInfo *RCIA, SILValue EnumValue,
SILBasicBlock *BB) {
// Otherwise, see if we can find a retain_value or strong_retain associated
// with that enum in the relevant predecessor.
auto FirstInc = std::find_if(
BB->rbegin(), BB->rend(),
[&RCIA, &EnumValue](const SILInstruction &I) -> bool {
// If I is not an increment, ignore it.
if (!isa<StrongRetainInst>(I) && !isa<RetainValueInst>(I))
return false;
// Otherwise, if the increments operand stripped of RC identity
// preserving
// ops matches EnumValue, it is the first increment we are interested
// in.
return EnumValue == RCIA->getRCIdentityRoot(I.getOperand(0));
});
// If we do not find a ref count increment in the relevant BB, skip this
// enum since there is nothing we can do.
if (FirstInc == BB->rend())
return nullptr;
// Otherwise, see if there are any instructions in between FirstPredInc and
// the end of the given basic block that could decrement first pred. If such
// an instruction exists, we cannot perform this optimization so continue.
if (valueHasARCDecrementOrCheckInInstructionRange(
EnumValue, (*FirstInc).getIterator(),
BB->getTerminator()->getIterator(), AA))
return nullptr;
return &*FirstInc;
}
static bool findRetainsSinkableFromSwitchRegionForEnum(
AliasAnalysis *AA, RCIdentityFunctionInfo *RCIA, SILValue EnumValue,
EnumBBCaseList &Map, SmallVectorImpl<SILInstruction *> &DeleteList) {
// For each predecessor with argument type...
for (auto &P : Map) {
SILBasicBlock *PredBB = P.first;
EnumElementDecl *Decl = P.second;
// If the case does not have an argument type, skip the predecessor since
// there will not be a retain to sink.
if (!Decl->hasAssociatedValues())
continue;
// Ok, we found a payloaded predecessor. Look backwards through the
// predecessor for the first ref count increment on EnumValue. If there
// are no ref count decrements in between the increment and the terminator
// of the BB, then we can sink the retain out of the switch enum.
auto *Inc = findLastSinkableMatchingEnumValueRCIncrementInPred(
AA, RCIA, EnumValue, PredBB);
// If we do not find such an increment, there is nothing we can do, bail.
if (!Inc)
return false;
// Otherwise add the increment to the delete list.
DeleteList.push_back(Inc);
}
// If we were able to process each predecessor successfully, return true.
return true;
}
bool BBEnumTagDataflowState::sinkIncrementsOutOfSwitchRegions(
AliasAnalysis *AA, RCIdentityFunctionInfo *RCIA) {
bool Changed = false;
unsigned NumPreds = std::distance(getBB()->pred_begin(), getBB()->pred_end());
llvm::SmallVector<SILInstruction *, 4> DeleteList;
// For each (EnumValue, [(BB, EnumTag)]) that we are tracking...
for (auto &P : EnumToEnumBBCaseListMap) {
// Clear our delete list.
DeleteList.clear();
// If EnumValue is null, we deleted this entry. There is nothing to do for
// this value... Skip it.
if (!P.has_value())
continue;
// Look up the actual enum value using our index to make sure that other
// parts of the pass have not destroyed the value. In such a case, just
// continue.
SILValue EnumValue = getContext().getValueForID(P->first);
if (!EnumValue)
continue;
EnumValue = RCIA->getRCIdentityRoot(EnumValue);
EnumBBCaseList &Map = P->second;
// If we do not have a tag associated with this enum value for each
// predecessor, we are not a switch region exit for this enum value. Skip
// this value.
if (Map.size() != NumPreds)
continue;
// Look through our predecessors for a set of ref count increments on our
// enum value for every payloaded case that *could* be sunk. If we miss an
// increment from any of the payloaded case there is nothing we can do here,
// so skip this enum value.
if (!findRetainsSinkableFromSwitchRegionForEnum(AA, RCIA, EnumValue, Map,
DeleteList))
continue;
// If we do not have any payload arguments, then we should have an empty
// delete list and there is nothing to do here.
if (DeleteList.empty())
continue;
// Ok, we can perform this transformation! Insert the new retain_value and
// delete all of the ref count increments from the predecessor BBs.
//
// TODO: Which debug loc should we use here? Using one of the locs from the
// delete list seems reasonable for now...
SILBuilder Builder(getBB()->begin());
Builder.createRetainValue(
DeleteList[0]->getLoc(), EnumValue,
cast<RefCountingInst>(DeleteList[0])->getAtomicity());
for (auto *I : DeleteList)
I->eraseFromParent();
++NumSunk;
Changed = true;
}
return Changed;
}
void BBEnumTagDataflowState::dump() const {
#ifndef NDEBUG
llvm::dbgs() << "Dumping state for BB" << BB.get()->getDebugID() << "\n";
llvm::dbgs() << "Block States:\n";
for (auto &P : ValueToCaseMap) {
if (!P) {
llvm::dbgs() << " Skipping blotted value.\n";
continue;
}
unsigned ID = P->first;
SILValue V = getContext().getValueForID(ID);
if (!V) {
llvm::dbgs() << " ID: " << ID << ". Value: BLOTTED.\n";
continue;
}
llvm::dbgs() << " ID: " << ID << ". Value: " << V;
}
llvm::dbgs() << "Predecessor States:\n";
// For each (EnumValue, [(BB, EnumTag)]) that we are tracking...
for (auto &P : EnumToEnumBBCaseListMap) {
if (!P) {
llvm::dbgs() << " Skipping blotted value.\n";
continue;
}
unsigned ID = P->first;
SILValue V = getContext().getValueForID(ID);
if (!V) {
llvm::dbgs() << " ID: " << ID << ". Value: BLOTTED.\n";
continue;
}
llvm::dbgs() << " ID: " << ID << ". Value: " << V;
llvm::dbgs() << " Case List:\n";
for (auto &P2 : P->second) {
llvm::dbgs() << " BB" << P2.first->getDebugID() << ": ";
P2.second->dump(llvm::dbgs());
llvm::dbgs() << "\n";
}
llvm::dbgs() << " End Case List.\n";
}
#endif
}
bool BBEnumTagDataflowState::init(EnumCaseDataflowContext &NewContext,
SILBasicBlock *NewBB) {
assert(NewBB && "NewBB should not be null");
Context = &NewContext;
BB = NewBB;
return true;
}
//===----------------------------------------------------------------------===//
// Generic Sinking Code
//===----------------------------------------------------------------------===//
/// Hoist release on a SILArgument to its predecessors.
static bool hoistSILArgumentReleaseInst(SILBasicBlock *BB) {
// There is no block to hoist releases to.
if (BB->pred_empty())
return false;
// Only try to hoist the first instruction. RRCM should have hoisted the
// release
// to the beginning of the block if it can.
auto Head = &*BB->begin();
// Make sure it is a release instruction.
if (!isReleaseInstruction(&*Head))
return false;
// Make sure it is a release on a SILArgument of the current basic block..
auto *SA = dyn_cast<SILArgument>(Head->getOperand(0));
if (!SA || SA->getParent() != BB)
return false;
// Make sure the release will not be blocked by the terminator instructions
// Make sure the terminator does not block, nor is a branch with multiple
// targets.
for (auto P : BB->getPredecessorBlocks()) {
if (!isa<BranchInst>(P->getTerminator()))
return false;
}
// Make sure we can get all the incoming values.
llvm::SmallVector<SILValue, 4> PredValues;
if (!SA->getIncomingPhiValues(PredValues))
return false;
// Ok, we can get all the incoming values and create releases for them.
unsigned indices = 0;
for (auto P : BB->getPredecessorBlocks()) {
createDecrementBefore(PredValues[indices++], P->getTerminator());
}
// Erase the old instruction.
Head->eraseFromParent();
++NumSILArgumentReleaseHoisted;
return true;
}
static const int SinkSearchWindow = 6;
/// Returns True if we can sink this instruction to another basic block.
static bool canSinkInstruction(SILInstruction *Inst) {
if (hasOwnershipOperandsOrResults(Inst))
return false;
return !Inst->hasUsesOfAnyResult() && !isa<TermInst>(Inst);
}
/// Returns true if this instruction is a skip barrier, which means that
/// we can't sink other instructions past it.
static bool isSinkBarrier(SILInstruction *Inst) {
if (isa<TermInst>(Inst))
return false;
if (Inst->mayHaveSideEffects())
return true;
return false;
}
using ValueInBlock = std::pair<SILValue, SILBasicBlock *>;
using ValueToBBArgIdxMap = llvm::DenseMap<ValueInBlock, int>;
enum OperandRelation {
/// Uninitialized state.
NotDeterminedYet,
/// The original operand values are equal.
AlwaysEqual,
/// The operand values are equal after replacing with the successor block
/// arguments.
EqualAfterMove
};
/// Find a root value for operand \p In. This function inspects a sil
/// value and strips trivial conversions such as values that are passed
/// as arguments to basic blocks with a single predecessor or type casts.
/// This is a shallow one-step search and not a deep recursive search.
///
/// For example, in the SIL code below, the root of %10 is %3, because it is
/// the only possible incoming value.
///
/// bb1:
/// %3 = unchecked_enum_data %0 : $Optional<X>, #Optional.Some!enumelt
/// checked_cast_br [exact] X in %3 : $X to $X, bb4, bb5 // id: %4
///
/// bb4(%10 : $X): // Preds: bb1
/// strong_release %10 : $X
/// br bb2
///
static SILValue findValueShallowRoot(const SILValue &In) {
// If this is a basic block argument with a single caller
// then we know exactly which value is passed to the argument.
if (auto *Arg = dyn_cast<SILArgument>(In)) {
SILBasicBlock *Parent = Arg->getParent();
SILBasicBlock *Pred = Parent->getSinglePredecessorBlock();
if (!Pred)
return In;
// If the terminator is a cast instruction then use the pre-cast value.
if (auto CCBI = dyn_cast<CheckedCastBranchInst>(Pred->getTerminator())) {
assert(CCBI->getSuccessBB() == Parent && "Inspecting the wrong block");
// In swift it is legal to cast non reference-counted references into
// object references. For example: func f(x : C.Type) -> Any {return x}
// Here we check that the uncasted reference is reference counted.
SILValue V = CCBI->getOperand();
if (V->getType().isReferenceCounted(Pred->getParent()->getModule())) {
return V;
}
}
// If the single predecessor terminator is a branch then the root is
// the argument to the terminator.
if (auto BI = dyn_cast<BranchInst>(Pred->getTerminator())) {
assert(BI->getDestBB() == Parent && "Invalid terminator");
unsigned Idx = Arg->getIndex();
return BI->getArg(Idx);
}
if (auto CBI = dyn_cast<CondBranchInst>(Pred->getTerminator())) {
return CBI->getArgForDestBB(Parent, Arg);
}
}
return In;
}
/// Search for an instruction that is identical to \p Iden by scanning
/// \p BB starting at the end of the block, stopping on sink barriers.
/// The \p opRelation must be consistent for all operand comparisons.
SILInstruction *findIdenticalInBlock(SILBasicBlock *BB, SILInstruction *Iden,
const ValueToBBArgIdxMap &valueToArgIdxMap,
OperandRelation &opRelation) {
int SkipBudget = SinkSearchWindow;
SILBasicBlock::iterator InstToSink = BB->getTerminator()->getIterator();
SILBasicBlock *IdenBlock = Iden->getParent();
// The compare function for instruction operands.
auto operandCompare = [&](const SILValue &Op1, const SILValue &Op2) -> bool {
if (opRelation != EqualAfterMove && Op1 == Op2) {
// The trivial case.
opRelation = AlwaysEqual;
return true;
}
// Check if both operand values are passed to the same block argument in the
// successor block. This means that the operands are equal after we move the
// instruction into the successor block.
if (opRelation != AlwaysEqual) {
auto Iter1 = valueToArgIdxMap.find({Op1, IdenBlock});
if (Iter1 != valueToArgIdxMap.end()) {
auto Iter2 = valueToArgIdxMap.find({Op2, BB});
if (Iter2 != valueToArgIdxMap.end() && Iter1->second == Iter2->second) {
opRelation = EqualAfterMove;
return true;
}
}
}
return false;
};
while (SkipBudget) {
// If we found a sinkable instruction that is identical to our goal
// then return it.
if (canSinkInstruction(&*InstToSink) &&
Iden->isIdenticalTo(&*InstToSink, operandCompare)) {
LLVM_DEBUG(llvm::dbgs() << "Found an identical instruction.");
return &*InstToSink;
}
// If this instruction is a skip-barrier end the scan.
if (isSinkBarrier(&*InstToSink))
return nullptr;
// If this is the first instruction in the block then we are done.
if (InstToSink == BB->begin())
return nullptr;
--SkipBudget;
InstToSink = std::prev(InstToSink);
LLVM_DEBUG(llvm::dbgs() << "Continuing scan. Next inst: " << *InstToSink);
}
return nullptr;
}
/// The 2 instructions given are not identical, but are passed as arguments
/// to a common successor. It may be cheaper to pass one of their operands
/// to the successor instead of the whole instruction.
/// Return None if no such operand could be found, otherwise return the index
/// of a suitable operand.
static std::optional<unsigned>
cheaperToPassOperandsAsArguments(SILInstruction *First,
SILInstruction *Second) {
// This will further enable to sink strong_retain_unowned instructions,
// which provides more opportunities for the unowned-optimization in
// LLVMARCOpts.
#define LOADABLE_REF_STORAGE(Name, ...) \
if (isa<Name##ToRefInst>(First) && isa<Name##ToRefInst>(Second)) { \
return 0; \
}
#include "swift/AST/ReferenceStorage.def"
// TODO: Add more cases than Struct
auto *FirstStruct = dyn_cast<StructInst>(First);
auto *SecondStruct = dyn_cast<StructInst>(Second);
if (!FirstStruct || !SecondStruct)
return std::nullopt;
assert(FirstStruct->getNumOperands() == SecondStruct->getNumOperands() &&
FirstStruct->getType() == SecondStruct->getType() &&
"Types should be identical");
std::optional<unsigned> DifferentOperandIndex;
// Check operands.
for (unsigned i = 0, e = First->getNumOperands(); i != e; ++i) {
if (FirstStruct->getOperand(i) != SecondStruct->getOperand(i)) {
// Only track one different operand for now
if (DifferentOperandIndex)
return std::nullopt;
DifferentOperandIndex = i;
}
}
if (!DifferentOperandIndex)
return std::nullopt;
// Found a different operand, now check to see if its type is something
// cheap enough to sink.
// TODO: Sink more than just integers.
SILType ArgTy = FirstStruct->getOperand(*DifferentOperandIndex)->getType();
if (!ArgTy.is<BuiltinIntegerType>())
return std::nullopt;
return *DifferentOperandIndex;
}
/// Return the value that's passed from block \p From to block \p To
/// (if there is a branch between From and To) as the Nth argument.
SILValue getArgForBlock(SILBasicBlock *From, SILBasicBlock *To,
unsigned ArgNum) {
TermInst *Term = From->getTerminator();
if (auto *CondBr = dyn_cast<CondBranchInst>(Term)) {
if (CondBr->getFalseBB() == To)
return CondBr->getFalseArgs()[ArgNum];
if (CondBr->getTrueBB() == To)
return CondBr->getTrueArgs()[ArgNum];
}
if (auto *Br = dyn_cast<BranchInst>(Term))
return Br->getArg(ArgNum);
return SILValue();
}
// Try to sink values from the Nth argument \p ArgNum.
static bool sinkLiteralArguments(SILBasicBlock *BB, unsigned ArgNum) {
assert(ArgNum < BB->getNumArguments() && "Invalid argument");
// Check if the argument passed to the first predecessor is a literal inst.
SILBasicBlock *FirstPred = *BB->pred_begin();
SILValue FirstArg = getArgForBlock(FirstPred, BB, ArgNum);
LiteralInst *FirstLiteral = dyn_cast_or_null<LiteralInst>(FirstArg);
if (!FirstLiteral)
return false;
// Check if the Nth argument in all predecessors is identical.
for (auto P : BB->getPredecessorBlocks()) {
if (P == FirstPred)
continue;
// Check that the incoming value is identical to the first literal.
SILValue PredArg = getArgForBlock(P, BB, ArgNum);
LiteralInst *PredLiteral = dyn_cast_or_null<LiteralInst>(PredArg);
if (!PredLiteral || !PredLiteral->isIdenticalTo(FirstLiteral))
return false;
}
// Replace the use of the argument with the cloned literal.
auto Cloned = FirstLiteral->clone(&*BB->begin());
BB->getArgument(ArgNum)->replaceAllUsesWith(Cloned);
return true;
}
// Try to sink values from the Nth argument \p ArgNum.
static bool sinkArgument(EnumCaseDataflowContext &Context, SILBasicBlock *BB, unsigned ArgNum) {
assert(ArgNum < BB->getNumArguments() && "Invalid argument");
// Find the first predecessor, the first terminator and the Nth argument.
SILBasicBlock *FirstPred = *BB->pred_begin();
TermInst *FirstTerm = FirstPred->getTerminator();
auto FirstPredArg = FirstTerm->getOperand(ArgNum);
auto *FSI = dyn_cast<SingleValueInstruction>(FirstPredArg);
// TODO: MultiValueInstruction?
// We only move instructions with a single use.
if (!FSI || !hasOneNonDebugUse(FSI))
return false;
if (hasOwnershipOperandsOrResults(FSI))
return false;
// Even if the incoming instruction has no ownership, the argument may have.
// This can happen with enums which are constructed with a non-payload case:
//
// %1 = enum $Optional<C>, #Optional.none!enumelt
// br bb3(%1)
// bb1(%3 : @owned $Optional<C>):
//
if (BB->getArgument(ArgNum)->getOwnershipKind() != OwnershipKind::None)
return false;
// The list of identical instructions.
SmallVector<SingleValueInstruction *, 8> Clones;
Clones.push_back(FSI);
// Don't move instructions that are sensitive to their location.
//
// If this instruction can read memory, we try to be conservatively not to
// move it, as there may be instructions that can clobber the read memory
// from current place to the place where it is moved to.
if (FSI->mayReadFromMemory() ||
(FSI->mayHaveSideEffects() && !isa<AllocationInst>(FSI)))
return false;
// If the instructions are different, but only in terms of a cheap operand
// then we can still sink it, and create new arguments for this operand.
std::optional<unsigned> DifferentOperandIndex;
// Check if the Nth argument in all predecessors is identical.
for (auto P : BB->getPredecessorBlocks()) {
if (P == FirstPred)
continue;
// Only handle branch or conditional branch instructions.
TermInst *TI = P->getTerminator();
if (!isa<BranchInst>(TI) && !isa<CondBranchInst>(TI))
return false;
// Find the Nth argument passed to BB.
SILValue Arg = TI->getOperand(ArgNum);
// If it's not the same basic kind of node, neither isIdenticalTo nor
// cheaperToPassOperandsAsArguments will return true.
if (Arg->getKind() != FSI->getValueKind())
return false;
// Since it's the same kind, Arg must also be a single-value instruction.
auto *SI = cast<SingleValueInstruction>(Arg);
if (!hasOneNonDebugUse(SI))
return false;
if (SI->isIdenticalTo(FSI)) {
Clones.push_back(SI);
continue;
}
// If the instructions are close enough, then we should sink them anyway.
// For example, we should sink 'struct S(%0)' if %0 is small, eg, an integer
auto MaybeDifferentOp = cheaperToPassOperandsAsArguments(FSI, SI);
// Couldn't find a suitable operand, so bail.
if (!MaybeDifferentOp)
return false;
unsigned DifferentOp = *MaybeDifferentOp;
// Make sure we found the same operand as prior iterations.
if (DifferentOperandIndex && DifferentOp != *DifferentOperandIndex)
return false;
DifferentOperandIndex = DifferentOp;
Clones.push_back(SI);
}
auto *Undef = SILUndef::get(FSI);
// Delete the debug info of the instruction that we are about to sink.
deleteAllDebugUses(FSI);
if (DifferentOperandIndex) {
// Sink one of the instructions to BB
FSI->moveBefore(&*BB->begin());
// The instruction we are lowering has an argument which is different
// for each predecessor. We need to sink the instruction, then add
// arguments for each predecessor.
BB->getArgument(ArgNum)->replaceAllUsesWith(FSI);
const auto &ArgType = FSI->getOperand(*DifferentOperandIndex)->getType();
BB->replacePhiArgument(ArgNum, ArgType, OwnershipKind::Owned);
// Update all branch instructions in the predecessors to pass the new
// argument to this BB.
auto CloneIt = Clones.begin();
for (auto P : BB->getPredecessorBlocks()) {
// Only handle branch or conditional branch instructions.
TermInst *TI = P->getTerminator();
assert((isa<BranchInst>(TI) || isa<CondBranchInst>(TI)) &&
"Branch instruction required");
// TODO: MultiValueInstruction
auto *CloneInst = *CloneIt;
TI->setOperand(ArgNum, CloneInst->getOperand(*DifferentOperandIndex));
// Now delete the clone as we only needed it operand.
if (CloneInst != FSI)
eliminateDeadInstruction(CloneInst);
++CloneIt;
}
assert(CloneIt == Clones.end() && "Clone/pred mismatch");
// The sunk instruction should now read from the argument of the BB it
// was moved to.
FSI->setOperand(*DifferentOperandIndex, BB->getArgument(ArgNum));
return true;
}
// Sink one of the copies of the instruction.
FSI->replaceAllUsesWithUndef();
FSI->moveBefore(&*BB->begin());
BB->getArgument(ArgNum)->replaceAllUsesWith(FSI);
// The argument is no longer in use. Replace all incoming inputs with undef
// and try to delete the instruction.
for (auto S : Clones) {
if (S != FSI) {
deleteAllDebugUses(S);
S->replaceAllUsesWith(Undef);
auto DeadArgInst = cast<SILInstruction>(S);
for (SILValue Result : DeadArgInst->getResults()) {
Context.blotValue(Result);
}
DeadArgInst->eraseFromParent();
}
}
return true;
}
/// Try to sink literals that are passed to arguments that are coming from
/// multiple predecessors.
/// Notice that unlike other sinking methods in this file we do allow sinking
/// of literals from blocks with multiple successors.
static bool sinkLiteralsFromPredecessors(SILBasicBlock *BB) {
if (BB->pred_empty() || BB->getSinglePredecessorBlock())
return false;
// Try to sink values from each of the arguments to the basic block.
bool Changed = false;
for (int i = 0, e = BB->getNumArguments(); i < e; ++i)
Changed |= sinkLiteralArguments(BB, i);
return Changed;
}
/// Try to sink identical arguments coming from multiple predecessors.
static bool sinkArgumentsFromPredecessors(EnumCaseDataflowContext &Context,
SILBasicBlock *BB) {
if (BB->pred_empty() || BB->getSinglePredecessorBlock())
return false;
// This block must be the only successor of all the predecessors.
for (auto P : BB->getPredecessorBlocks())
if (P->getSingleSuccessorBlock() != BB)
return false;
// Try to sink values from each of the arguments to the basic block.
bool Changed = false;
for (int i = 0, e = BB->getNumArguments(); i < e; ++i)
Changed |= sinkArgument(Context, BB, i);
return Changed;
}
/// canonicalize retain/release instructions and make them amenable to
/// sinking by selecting canonical pointers. We reduce the number of possible
/// inputs by replacing values that are unlikely to be a canonical values.
/// Reducing the search space increases the chances of matching ref count
/// instructions to one another and the chance of sinking them. We replace
/// values that come from basic block arguments with the caller values and
/// strip casts.
static bool canonicalizeRefCountInstrs(SILBasicBlock *BB) {
bool Changed = false;
for (auto I = BB->begin(), E = BB->end(); I != E; ++I) {
if (!isa<StrongReleaseInst>(I) && !isa<StrongRetainInst>(I))
continue;
SILValue Ref = I->getOperand(0);
SILValue Root = findValueShallowRoot(Ref);
if (Ref != Root) {
I->setOperand(0, Root);
Changed = true;
}
}
return Changed;
}
static bool sinkCodeFromPredecessors(EnumCaseDataflowContext &Context,
SILBasicBlock *BB) {
bool Changed = false;
if (BB->pred_empty())
return Changed;
// This block must be the only successor of all the predecessors.
for (auto P : BB->getPredecessorBlocks())
if (P->getSingleSuccessorBlock() != BB)
return Changed;
SILBasicBlock *FirstPred = *BB->pred_begin();
// The first Pred must have at least one non-terminator.
if (FirstPred->getTerminator() == &*FirstPred->begin())
return Changed;
LLVM_DEBUG(llvm::dbgs() << " Sinking values from predecessors.\n");
// Map values in predecessor blocks to argument indices of the successor
// block. For example:
//
// bb1:
// br bb3(%a, %b) // %a -> 0, %b -> 1
// bb2:
// br bb3(%c, %d) // %c -> 0, %d -> 1
// bb3(%x, %y):
// ...
ValueToBBArgIdxMap valueToArgIdxMap;
for (auto P : BB->getPredecessorBlocks()) {
if (auto *BI = dyn_cast<BranchInst>(P->getTerminator())) {
auto Args = BI->getArgs();
for (size_t idx = 0, size = Args.size(); idx < size; ++idx) {
valueToArgIdxMap[{Args[idx], P}] = idx;
}
}
}
unsigned SkipBudget = SinkSearchWindow;
// Start scanning backwards from the terminator.
auto InstToSink = FirstPred->getTerminator()->getIterator();
while (SkipBudget) {
LLVM_DEBUG(llvm::dbgs() << "Processing: " << *InstToSink);
// Save the duplicated instructions in case we need to remove them.
SmallVector<SILInstruction *, 4> Dups;
if (canSinkInstruction(&*InstToSink)) {
OperandRelation opRelation = NotDeterminedYet;
// For all preds:
for (auto P : BB->getPredecessorBlocks()) {
if (P == FirstPred)
continue;
// Search the duplicated instruction in the predecessor.
if (SILInstruction *DupInst = findIdenticalInBlock(
P, &*InstToSink, valueToArgIdxMap, opRelation)) {
Dups.push_back(DupInst);
} else {
LLVM_DEBUG(llvm::dbgs() << "Instruction mismatch.\n");
Dups.clear();
break;
}
}
// If we found duplicated instructions, sink one of the copies and delete
// the rest.
if (Dups.size()) {
LLVM_DEBUG(llvm::dbgs() << "Moving: " << *InstToSink);
InstToSink->moveBefore(&*BB->begin());
if (opRelation == EqualAfterMove) {
// Replace operand values (which are passed to the successor block)
// with corresponding block arguments.
for (size_t idx = 0, numOps = InstToSink->getNumOperands();
idx < numOps; ++idx) {
ValueInBlock OpInFirstPred(InstToSink->getOperand(idx), FirstPred);
assert(valueToArgIdxMap.count(OpInFirstPred) != 0);
int argIdx = valueToArgIdxMap[OpInFirstPred];
InstToSink->setOperand(idx, BB->getArgument(argIdx));
}
}
Changed = true;
for (auto I : Dups) {
I->replaceAllUsesPairwiseWith(&*InstToSink);
for (SILValue Result : I->getResults()) {
Context.blotValue(Result);
}
I->eraseFromParent();
++NumSunk;
}
// Restart the scan.
InstToSink = FirstPred->getTerminator()->getIterator();
LLVM_DEBUG(llvm::dbgs() << "Restarting scan. Next inst: "
<< *InstToSink);
continue;
}
}
// If this instruction was a barrier then we can't sink anything else.
if (isSinkBarrier(&*InstToSink)) {
LLVM_DEBUG(llvm::dbgs() << "Aborting on barrier: " << *InstToSink);
return Changed;
}
// This is the first instruction, we are done.
if (InstToSink == FirstPred->begin()) {
LLVM_DEBUG(llvm::dbgs() << "Reached the first instruction.");
return Changed;
}
--SkipBudget;
InstToSink = std::prev(InstToSink);
LLVM_DEBUG(llvm::dbgs() << "Continuing scan. Next inst: " << *InstToSink);
}
return Changed;
}
/// Sink retain_value, release_value before switch_enum to be retain_value,
/// release_value on the payload of the switch_enum in the destination BBs. We
/// only do this if the destination BBs have only the switch enum as its
/// predecessor.
static bool tryToSinkRefCountAcrossSwitch(SwitchEnumInst *Switch,
SILBasicBlock::iterator RV,
AliasAnalysis *AA,
RCIdentityFunctionInfo *RCIA) {
// If this instruction is not a retain_value, there is nothing left for us to
// do... bail...
if (!isa<RetainValueInst>(RV))
return false;
SILValue Ptr = RV->getOperand(0);
// Next go over all instructions after I in the basic block. If none of them
// can decrement our ptr value, we can move the retain over the ref count
// inst. If any of them do potentially decrement the ref count of Ptr, we can
// not move it.
auto SwitchIter = Switch->getIterator();
if (auto B = valueHasARCDecrementOrCheckInInstructionRange(Ptr, RV,
SwitchIter, AA)) {
RV->moveBefore(&**B);
return true;
}
// If the retain value's argument is not the switch's argument, we can't do
// anything with our simplistic analysis... bail...
if (RCIA->getRCIdentityRoot(Ptr) !=
RCIA->getRCIdentityRoot(Switch->getOperand()))
return false;
// If the enum has a deinit, preserve the original release.
assert(!hasValueDeinit(Ptr) &&
"enum with deinit is not RC-identical to its payload");
// If S has a default case bail since the default case could represent
// multiple cases.
//
// TODO: I am currently just disabling this behavior so we can get this out
// for Seed 5. After Seed 5, we should be able to recognize if a switch_enum
// handles all cases except for 1 and has a default case. We might be able to
// stick code into SILBuilder that has this behavior.
if (Switch->hasDefault())
return false;
// Ok, we have a ref count instruction, sink it!
SILBuilderWithScope Builder(Switch, &*RV);
for (unsigned i = 0, e = Switch->getNumCases(); i != e; ++i) {
auto Case = Switch->getCase(i);
EnumElementDecl *Enum = Case.first;
SILBasicBlock *Succ = Case.second;
Builder.setInsertionPoint(&*Succ->begin());
if (Enum->hasAssociatedValues())
createRefCountOpForPayload(Builder, &*RV, Enum, Switch->getOperand());
}
RV->eraseFromParent();
++NumSunk;
return true;
}
/// Sink retain_value, release_value before select_enum to be retain_value,
/// release_value on the payload of the switch_enum in the destination BBs. We
/// only do this if the destination BBs have only the switch enum as its
/// predecessor.
static bool tryToSinkRefCountAcrossSelectEnum(CondBranchInst *CondBr,
SILBasicBlock::iterator I,
AliasAnalysis *AA,
RCIdentityFunctionInfo *RCIA) {
// If this instruction is not a retain_value, there is nothing left for us to
// do... bail...
if (!isa<RetainValueInst>(I))
return false;
// Make sure the condition comes from a select_enum
auto *SEI = dyn_cast<SelectEnumInst>(CondBr->getCondition());
if (!SEI)
return false;
// Try to find a single literal "true" case.
// TODO: More general conditions in which we can relate the BB to a single
// case, such as when there's a single literal "false" case.
NullablePtr<EnumElementDecl> TrueElement = SEI->getSingleTrueElement();
if (TrueElement.isNull())
return false;
// Next go over all instructions after I in the basic block. If none of them
// can decrement our ptr value, we can move the retain over the ref count
// inst. If any of them do potentially decrement the ref count of Ptr, we can
// not move it.
SILValue Ptr = I->getOperand(0);
auto CondBrIter = CondBr->getIterator();
if (auto B = valueHasARCDecrementOrCheckInInstructionRange(Ptr, std::next(I),
CondBrIter, AA)) {
I->moveBefore(&**B);
return false;
}
// If the retain value's argument is not the cond_br's argument, we can't do
// anything with our simplistic analysis... bail...
if (RCIA->getRCIdentityRoot(Ptr) !=
RCIA->getRCIdentityRoot(SEI->getEnumOperand()))
return false;
// If the enum has a deinit, preserve the original release.
assert(!hasValueDeinit(Ptr) &&
"enum with deinit is not RC-identical to its payload");
// Work out which enum element is the true branch, and which is false.
// If the enum only has 2 values and its tag isn't the true branch, then we
// know the true branch must be the other tag.
EnumElementDecl *Elts[2] = {TrueElement.get(), nullptr};
EnumDecl *E = SEI->getEnumOperand()->getType().getEnumOrBoundGenericEnum();
if (!E)
return false;
// Look for a single other element on this enum.
EnumElementDecl *OtherElt = nullptr;
for (EnumElementDecl *Elt : E->getAllElements()) {
// Skip the case where we find the select_enum element
if (Elt == TrueElement.get())
continue;
// If we find another element, then we must have more than 2, so bail.
if (OtherElt)
return false;
OtherElt = Elt;
}
// Only a single enum element? How would this even get here? We should
// handle it in SILCombine.
if (!OtherElt)
return false;
Elts[1] = OtherElt;
SILBuilderWithScope Builder(SEI, &*I);
// Ok, we have a ref count instruction, sink it!
for (unsigned i = 0; i != 2; ++i) {
EnumElementDecl *Enum = Elts[i];
SILBasicBlock *Succ = i == 0 ? CondBr->getTrueBB() : CondBr->getFalseBB();
Builder.setInsertionPoint(&*Succ->begin());
if (Enum->hasAssociatedValues())
createRefCountOpForPayload(Builder, &*I, Enum, SEI->getEnumOperand());
}
I->eraseFromParent();
++NumSunk;
return true;
}
static bool tryTosinkIncrementsIntoSwitchRegions(SILBasicBlock::iterator T,
SILBasicBlock::iterator I,
bool CanSinkToSuccessors,
AliasAnalysis *AA,
RCIdentityFunctionInfo *RCIA) {
// The following methods should only be attempted if we can sink to our
// successor.
if (CanSinkToSuccessors) {
// If we have a switch, try to sink ref counts across it and then return
// that result. We do not keep processing since the code below cannot
// properly sink ref counts over switch_enums so we might as well exit
// early.
if (auto *S = dyn_cast<SwitchEnumInst>(T))
return tryToSinkRefCountAcrossSwitch(S, I, AA, RCIA);
// In contrast, even if we do not sink ref counts across a cond_br from a
// select_enum, we may be able to sink anyways. So we do not return on a
// failure case.
if (auto *CondBr = dyn_cast<CondBranchInst>(T))
if (tryToSinkRefCountAcrossSelectEnum(CondBr, I, AA, RCIA))
return true;
}
// At this point, this is a retain on a regular SSA value, leave it to retain
// release code motion to sink.
return false;
}
/// Try sink a retain as far as possible. This is either to successor BBs,
/// or as far down the current BB as possible
static bool sinkIncrementsIntoSwitchRegions(SILBasicBlock *BB,
AliasAnalysis *AA,
RCIdentityFunctionInfo *RCIA) {
// Make sure that each one of our successors only has one predecessor,
// us.
// If that condition is not true, we can still sink to the end of this BB,
// but not to successors.
bool CanSinkToSuccessor = std::none_of(
BB->succ_begin(), BB->succ_end(), [](const SILSuccessor &S) -> bool {
SILBasicBlock *SuccBB = S.getBB();
return !SuccBB || !SuccBB->getSinglePredecessorBlock();
});
SILInstruction *S = BB->getTerminator();
auto SI = S->getIterator(), SE = BB->begin();
if (SI == SE)
return false;
bool Changed = false;
// Walk from the terminator up the BB. Try move retains either to the next
// BB, or the end of this BB. Note that ordering is maintained of retains
// within this BB.
SI = std::prev(SI);
while (SI != SE) {
SILInstruction *Inst = &*SI;
SI = std::prev(SI);
// Try to:
//
// 1. If there are no decrements between our ref count inst and
// terminator, sink the ref count inst into either our successors.
// 2. If there are such decrements, move the retain right before that
// decrement.
Changed |= tryTosinkIncrementsIntoSwitchRegions(
S->getIterator(), Inst->getIterator(), CanSinkToSuccessor, AA, RCIA);
}
// Handle the first instruction in the BB.
Changed |= tryTosinkIncrementsIntoSwitchRegions(S->getIterator(), SI,
CanSinkToSuccessor, AA, RCIA);
return Changed;
}
//===----------------------------------------------------------------------===//
// Top Level Driver
//===----------------------------------------------------------------------===//
static bool processFunction(SILFunction *F, AliasAnalysis *AA,
PostOrderFunctionInfo *PO,
RCIdentityFunctionInfo *RCIA,
bool HoistReleases) {
bool Changed = false;
EnumCaseDataflowContext BBToStateMap(PO);
for (unsigned RPOIdx = 0, RPOEnd = BBToStateMap.size(); RPOIdx < RPOEnd;
++RPOIdx) {
LLVM_DEBUG(llvm::dbgs() << "Visiting BB RPO#" << RPOIdx << "\n");
BBEnumTagDataflowState &State = BBToStateMap.getRPOState(RPOIdx);
LLVM_DEBUG(llvm::dbgs() <<" Predecessors (empty if no predecessors):\n");
LLVM_DEBUG(for (SILBasicBlock *Pred
: State.getBB()->getPredecessorBlocks()) {
llvm::dbgs() << " BB#" << RPOIdx << "; Ptr: " << Pred << "\n";
});
LLVM_DEBUG(llvm::dbgs() << " State Addr: " << &State << "\n");
// Merge in our predecessor states. We relook up our the states for our
// predecessors to avoid memory invalidation issues due to copying in the
// dense map.
LLVM_DEBUG(llvm::dbgs() << " Merging predecessors!\n");
State.mergePredecessorStates();
if (!F->hasOwnership()) {
// If our predecessors cover any of our enum values, attempt to hoist
// releases up the CFG onto enum payloads or sink retains out of switch
// regions.
LLVM_DEBUG(llvm::dbgs() << " Attempting to move releases into "
"predecessors!\n");
// Perform a relatively local forms of retain sinking and release hoisting
// regarding switch regions and SILargument. This are not handled by retain
// release code motion.
if (HoistReleases) {
Changed |= State.hoistDecrementsIntoSwitchRegions(AA);
}
// Sink switch related retains.
Changed |= sinkIncrementsIntoSwitchRegions(State.getBB(), AA, RCIA);
Changed |= State.sinkIncrementsOutOfSwitchRegions(AA, RCIA);
// Then attempt to sink code from predecessors. This can include retains
// which is why we always attempt to move releases up the CFG before sinking
// code from predecessors. We will never sink the hoisted releases from
// predecessors since the hoisted releases will be on the enum payload
// instead of the enum itself.
Changed |= canonicalizeRefCountInstrs(State.getBB());
}
Changed |= sinkCodeFromPredecessors(BBToStateMap, State.getBB());
Changed |= sinkArgumentsFromPredecessors(BBToStateMap, State.getBB());
Changed |= sinkLiteralsFromPredecessors(State.getBB());
if (!F->hasOwnership()) {
// Try to hoist release of a SILArgument to predecessors.
Changed |= hoistSILArgumentReleaseInst(State.getBB());
// Then perform the dataflow.
LLVM_DEBUG(llvm::dbgs() << " Performing the dataflow!\n");
Changed |= State.process();
}
}
return Changed;
}
class SILCodeMotion : public SILFunctionTransform {
bool HoistReleases;
public:
SILCodeMotion(bool TryReleaseHoisting) : HoistReleases(TryReleaseHoisting) {}
/// The entry point to the transformation.
void run() override {
auto *F = getFunction();
auto *AA = getAnalysis<AliasAnalysis>(F);
auto *PO = getAnalysis<PostOrderAnalysis>()->get(F);
auto *RCIA = getAnalysis<RCIdentityAnalysis>()->get(getFunction());
LLVM_DEBUG(llvm::dbgs() << "***** CodeMotion on function: " << F->getName()
<< " *****\n");
if (processFunction(F, AA, PO, RCIA, HoistReleases))
invalidateAnalysis(SILAnalysis::InvalidationKind::Instructions);
}
};
} // end anonymous namespace
/// Code motion that does not releases into diamonds.
SILTransform *swift::createEarlyCodeMotion() {
return new SILCodeMotion(false);
}
/// Code motion that hoists releases into diamonds.
SILTransform *swift::createLateCodeMotion() {
return new SILCodeMotion(true);
}
Reference in New Issue View Git Blame Copy Permalink